/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


/*
 * Deimos - cryptographic acceleration based upon Broadcom 582x.
 */

#include <sys/types.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/devops.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/cmn_err.h>
#include <sys/varargs.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/kmem.h>
#include <sys/ioccom.h>
#include <sys/open.h>
#include <sys/cred.h>
#include <sys/kstat.h>
#include <sys/strsun.h>
#include <sys/note.h>
#include <sys/crypto/common.h>
#include <sys/crypto/spi.h>
#include <sys/ddifm.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/fm/io/ddi.h>
#include <sys/crypto/dca.h>

/*
 * Core Deimos driver.
 */

static void             dca_enlist2(dca_listnode_t *, dca_listnode_t *,
    kmutex_t *);
static void             dca_rmlist2(dca_listnode_t *node, kmutex_t *);
static dca_listnode_t   *dca_delist2(dca_listnode_t *q, kmutex_t *);
static void             dca_free_context_list(dca_t *dca);
static int              dca_free_context_low(crypto_ctx_t *ctx);
static int              dca_attach(dev_info_t *, ddi_attach_cmd_t);
static int              dca_detach(dev_info_t *, ddi_detach_cmd_t);
static int              dca_suspend(dca_t *);
static int              dca_resume(dca_t *);
static int              dca_init(dca_t *);
static int              dca_reset(dca_t *, int);
static int              dca_initworklist(dca_t *, dca_worklist_t *);
static void             dca_uninit(dca_t *);
static void             dca_initq(dca_listnode_t *);
static void             dca_enqueue(dca_listnode_t *, dca_listnode_t *);
static dca_listnode_t   *dca_dequeue(dca_listnode_t *);
static dca_listnode_t   *dca_unqueue(dca_listnode_t *);
static dca_request_t    *dca_newreq(dca_t *);
static dca_work_t       *dca_getwork(dca_t *, int);
static void             dca_freework(dca_work_t *);
static dca_work_t       *dca_newwork(dca_t *);
static void             dca_destroywork(dca_work_t *);
static void             dca_schedule(dca_t *, int);
static void             dca_reclaim(dca_t *, int);
static uint_t           dca_intr(char *);
static void             dca_failure(dca_t *, ddi_fault_location_t,
                            dca_fma_eclass_t index, uint64_t, int, char *, ...);
static void             dca_jobtimeout(void *);
static int              dca_drain(dca_t *);
static void             dca_undrain(dca_t *);
static void             dca_rejectjobs(dca_t *);

#ifdef  SCHEDDELAY
static void             dca_schedtimeout(void *);
#endif

/*
 * Device operations.
 */
static struct dev_ops devops = {
        DEVO_REV,               /* devo_rev */
        0,                      /* devo_refcnt */
        nodev,                  /* devo_getinfo */
        nulldev,                /* devo_identify */
        nulldev,                /* devo_probe */
        dca_attach,             /* devo_attach */
        dca_detach,             /* devo_detach */
        nodev,                  /* devo_reset */
        NULL,                   /* devo_cb_ops */
        NULL,                   /* devo_bus_ops */
        ddi_power,              /* devo_power */
        ddi_quiesce_not_supported,      /* devo_quiesce */
};

#define IDENT           "PCI Crypto Accelerator"
#define IDENT_SYM       "Crypto Accel Sym 2.0"
#define IDENT_ASYM      "Crypto Accel Asym 2.0"

/* Space-padded, will be filled in dynamically during registration */
#define IDENT3  "PCI Crypto Accelerator Mod 2.0"

#define VENDOR  "Sun Microsystems, Inc."

#define STALETIME       (30 * SECOND)

#define crypto_prov_notify      crypto_provider_notification
                /* A 28 char function name doesn't leave much line space */

/*
 * Module linkage.
 */
static struct modldrv modldrv = {
        &mod_driverops,         /* drv_modops */
        IDENT,                  /* drv_linkinfo */
        &devops,                /* drv_dev_ops */
};

extern struct mod_ops mod_cryptoops;

static struct modlcrypto modlcrypto = {
        &mod_cryptoops,
        IDENT3
};

static struct modlinkage modlinkage = {
        MODREV_1,               /* ml_rev */
        &modldrv,               /* ml_linkage */
        &modlcrypto,
        NULL
};

/*
 * CSPI information (entry points, provider info, etc.)
 */

/* Mechanisms for the symmetric cipher provider */
static crypto_mech_info_t dca_mech_info_tab1[] = {
        /* DES-CBC */
        {SUN_CKM_DES_CBC, DES_CBC_MECH_INFO_TYPE,
            CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
            CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
            DES_KEY_LEN, DES_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
        /* 3DES-CBC */
        {SUN_CKM_DES3_CBC, DES3_CBC_MECH_INFO_TYPE,
            CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
            CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
            DES3_MIN_KEY_LEN, DES3_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES}
};

/* Mechanisms for the asymmetric cipher provider */
static crypto_mech_info_t dca_mech_info_tab2[] = {
        /* DSA */
        {SUN_CKM_DSA, DSA_MECH_INFO_TYPE,
            CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY |
            CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC,
            CRYPTO_BYTES2BITS(DSA_MIN_KEY_LEN),
            CRYPTO_BYTES2BITS(DSA_MAX_KEY_LEN),
            CRYPTO_KEYSIZE_UNIT_IN_BITS},

        /* RSA */
        {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
            CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
            CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
            CRYPTO_FG_VERIFY_RECOVER |
            CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
            CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
            CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
            CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
            CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
            CRYPTO_KEYSIZE_UNIT_IN_BITS},
        {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
            CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
            CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
            CRYPTO_FG_VERIFY_RECOVER |
            CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
            CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
            CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
            CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
            CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
            CRYPTO_KEYSIZE_UNIT_IN_BITS}
};

static void dca_provider_status(crypto_provider_handle_t, uint_t *);

static crypto_control_ops_t dca_control_ops = {
        dca_provider_status
};

static int dca_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_encrypt_update(crypto_ctx_t *, crypto_data_t *,
    crypto_data_t *, crypto_req_handle_t);
static int dca_encrypt_final(crypto_ctx_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
    crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);

static int dca_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_decrypt_update(crypto_ctx_t *, crypto_data_t *,
    crypto_data_t *, crypto_req_handle_t);
static int dca_decrypt_final(crypto_ctx_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
    crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);

static crypto_cipher_ops_t dca_cipher_ops = {
        dca_encrypt_init,
        dca_encrypt,
        dca_encrypt_update,
        dca_encrypt_final,
        dca_encrypt_atomic,
        dca_decrypt_init,
        dca_decrypt,
        dca_decrypt_update,
        dca_decrypt_final,
        dca_decrypt_atomic
};

static int dca_sign_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *,
    crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_sign_update(crypto_ctx_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_sign_final(crypto_ctx_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
    crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_sign_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_sign_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_sign_recover_atomic(crypto_provider_handle_t,
    crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
    crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);

static crypto_sign_ops_t dca_sign_ops = {
        dca_sign_init,
        dca_sign,
        dca_sign_update,
        dca_sign_final,
        dca_sign_atomic,
        dca_sign_recover_init,
        dca_sign_recover,
        dca_sign_recover_atomic
};

static int dca_verify_init(crypto_ctx_t *, crypto_mechanism_t *,
    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_verify_update(crypto_ctx_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_verify_final(crypto_ctx_t *, crypto_data_t *,
    crypto_req_handle_t);
static int dca_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
    crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_verify_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
static int dca_verify_recover(crypto_ctx_t *, crypto_data_t *,
    crypto_data_t *, crypto_req_handle_t);
static int dca_verify_recover_atomic(crypto_provider_handle_t,
    crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
    crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);

static crypto_verify_ops_t dca_verify_ops = {
        dca_verify_init,
        dca_verify,
        dca_verify_update,
        dca_verify_final,
        dca_verify_atomic,
        dca_verify_recover_init,
        dca_verify_recover,
        dca_verify_recover_atomic
};

static int dca_generate_random(crypto_provider_handle_t, crypto_session_id_t,
    uchar_t *, size_t, crypto_req_handle_t);

static crypto_random_number_ops_t dca_random_number_ops = {
        NULL,
        dca_generate_random
};

static int ext_info_sym(crypto_provider_handle_t prov,
    crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
static int ext_info_asym(crypto_provider_handle_t prov,
    crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
static int ext_info_base(crypto_provider_handle_t prov,
    crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id);

static crypto_provider_management_ops_t dca_provmanage_ops_1 = {
        ext_info_sym,           /* ext_info */
        NULL,                   /* init_token */
        NULL,                   /* init_pin */
        NULL                    /* set_pin */
};

static crypto_provider_management_ops_t dca_provmanage_ops_2 = {
        ext_info_asym,          /* ext_info */
        NULL,                   /* init_token */
        NULL,                   /* init_pin */
        NULL                    /* set_pin */
};

int dca_free_context(crypto_ctx_t *);

static crypto_ctx_ops_t dca_ctx_ops = {
        NULL,
        dca_free_context
};

/* Operations for the symmetric cipher provider */
static crypto_ops_t dca_crypto_ops1 = {
        &dca_control_ops,
        NULL,                           /* digest_ops */
        &dca_cipher_ops,
        NULL,                           /* mac_ops */
        NULL,                           /* sign_ops */
        NULL,                           /* verify_ops */
        NULL,                           /* dual_ops */
        NULL,                           /* cipher_mac_ops */
        NULL,                           /* random_number_ops */
        NULL,                           /* session_ops */
        NULL,                           /* object_ops */
        NULL,                           /* key_ops */
        &dca_provmanage_ops_1,          /* management_ops */
        &dca_ctx_ops
};

/* Operations for the asymmetric cipher provider */
static crypto_ops_t dca_crypto_ops2 = {
        &dca_control_ops,
        NULL,                           /* digest_ops */
        &dca_cipher_ops,
        NULL,                           /* mac_ops */
        &dca_sign_ops,
        &dca_verify_ops,
        NULL,                           /* dual_ops */
        NULL,                           /* cipher_mac_ops */
        &dca_random_number_ops,
        NULL,                           /* session_ops */
        NULL,                           /* object_ops */
        NULL,                           /* key_ops */
        &dca_provmanage_ops_2,          /* management_ops */
        &dca_ctx_ops
};

/* Provider information for the symmetric cipher provider */
static crypto_provider_info_t dca_prov_info1 = {
        CRYPTO_SPI_VERSION_1,
        NULL,                           /* pi_provider_description */
        CRYPTO_HW_PROVIDER,
        NULL,                           /* pi_provider_dev */
        NULL,                           /* pi_provider_handle */
        &dca_crypto_ops1,
        sizeof (dca_mech_info_tab1)/sizeof (crypto_mech_info_t),
        dca_mech_info_tab1,
        0,                              /* pi_logical_provider_count */
        NULL                            /* pi_logical_providers */
};

/* Provider information for the asymmetric cipher provider */
static crypto_provider_info_t dca_prov_info2 = {
        CRYPTO_SPI_VERSION_1,
        NULL,                           /* pi_provider_description */
        CRYPTO_HW_PROVIDER,
        NULL,                           /* pi_provider_dev */
        NULL,                           /* pi_provider_handle */
        &dca_crypto_ops2,
        sizeof (dca_mech_info_tab2)/sizeof (crypto_mech_info_t),
        dca_mech_info_tab2,
        0,                              /* pi_logical_provider_count */
        NULL                            /* pi_logical_providers */
};

/* Convenience macros */
#define DCA_SOFTC_FROM_CTX(ctx) ((dca_t *)(ctx)->cc_provider)
#define DCA_MECH_FROM_CTX(ctx) \
        (((dca_request_t *)(ctx)->cc_provider_private)->dr_ctx.ctx_cm_type)

static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
    caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
    dca_chain_t *head, int *n_chain);
static uint64_t dca_ena(uint64_t ena);
static caddr_t dca_bufdaddr_out(crypto_data_t *data);
static char *dca_fma_eclass_string(char *model, dca_fma_eclass_t index);
static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
    dca_fma_eclass_t eclass_index);

static void dca_fma_init(dca_t *dca);
static void dca_fma_fini(dca_t *dca);
static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
    const void *impl_data);


static dca_device_t dca_devices[] = {
        /* Broadcom vanilla variants */
        {       0x14e4, 0x5820, "Broadcom 5820" },
        {       0x14e4, 0x5821, "Broadcom 5821" },
        {       0x14e4, 0x5822, "Broadcom 5822" },
        {       0x14e4, 0x5825, "Broadcom 5825" },
        /* Sun specific OEMd variants */
        {       0x108e, 0x5454, "SCA" },
        {       0x108e, 0x5455, "SCA 1000" },
        {       0x108e, 0x5457, "SCA 500" },
        /* subsysid should be 0x5457, but got 0x1 from HW. Assume both here. */
        {       0x108e, 0x1, "SCA 500" },
};

/*
 * Device attributes.
 */
static struct ddi_device_acc_attr dca_regsattr = {
        DDI_DEVICE_ATTR_V1,
        DDI_STRUCTURE_LE_ACC,
        DDI_STRICTORDER_ACC,
        DDI_FLAGERR_ACC
};

static struct ddi_device_acc_attr dca_devattr = {
        DDI_DEVICE_ATTR_V0,
        DDI_STRUCTURE_LE_ACC,
        DDI_STRICTORDER_ACC
};

static struct ddi_device_acc_attr dca_bufattr = {
        DDI_DEVICE_ATTR_V0,
        DDI_NEVERSWAP_ACC,
        DDI_STRICTORDER_ACC
};

static struct ddi_dma_attr dca_dmaattr = {
        DMA_ATTR_V0,            /* dma_attr_version */
        0x0,                    /* dma_attr_addr_lo */
        0xffffffffUL,           /* dma_attr_addr_hi */
        0x00ffffffUL,           /* dma_attr_count_max */
        0x40,                   /* dma_attr_align */
        0x40,                   /* dma_attr_burstsizes */
        0x1,                    /* dma_attr_minxfer */
        0x00ffffffUL,           /* dma_attr_maxxfer */
        0xffffffffUL,           /* dma_attr_seg */
#if defined(__x86)
        512,                    /* dma_attr_sgllen */
#else
        1,                      /* dma_attr_sgllen */
#endif
        1,                      /* dma_attr_granular */
        DDI_DMA_FLAGERR         /* dma_attr_flags */
};

static void     *dca_state = NULL;
int     dca_mindma = 2500;

/*
 * FMA eclass string definitions. Note that these string arrays must be
 * consistent with the dca_fma_eclass_t enum.
 */
static char *dca_fma_eclass_sca1000[] = {
        "sca1000.hw.device",
        "sca1000.hw.timeout",
        "sca1000.none"
};

static char *dca_fma_eclass_sca500[] = {
        "sca500.hw.device",
        "sca500.hw.timeout",
        "sca500.none"
};

/*
 * DDI entry points.
 */
int
_init(void)
{
        int rv;

        DBG(NULL, DMOD, "dca: in _init");

        if ((rv = ddi_soft_state_init(&dca_state, sizeof (dca_t), 1)) != 0) {
                /* this should *never* happen! */
                return (rv);
        }

        if ((rv = mod_install(&modlinkage)) != 0) {
                /* cleanup here */
                ddi_soft_state_fini(&dca_state);
                return (rv);
        }

        return (0);
}

int
_fini(void)
{
        int rv;

        DBG(NULL, DMOD, "dca: in _fini");

        if ((rv = mod_remove(&modlinkage)) == 0) {
                /* cleanup here */
                ddi_soft_state_fini(&dca_state);
        }
        return (rv);
}

int
_info(struct modinfo *modinfop)
{
        DBG(NULL, DMOD, "dca: in _info");

        return (mod_info(&modlinkage, modinfop));
}

int
dca_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        ddi_acc_handle_t        pci;
        int                     instance;
        ddi_iblock_cookie_t     ibc;
        int                     intr_added = 0;
        dca_t                   *dca;
        ushort_t                venid;
        ushort_t                devid;
        ushort_t                revid;
        ushort_t                subsysid;
        ushort_t                subvenid;
        int                     i;
        int                     ret;
        char                    ID[64];
        static char             *unknowndev = "Unknown device";

#if DEBUG
        /* these are only used for debugging */
        ushort_t                pcicomm;
        ushort_t                pcistat;
        uchar_t                 cachelinesz;
        uchar_t                 mingnt;
        uchar_t                 maxlat;
        uchar_t                 lattmr;
#endif

        instance = ddi_get_instance(dip);

        DBG(NULL, DMOD, "dca: in dca_attach() for %d", instance);

        switch (cmd) {
        case DDI_RESUME:
                if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
                        dca_diperror(dip, "no soft state in detach");
                        return (DDI_FAILURE);
                }
                /* assumption: we won't be DDI_DETACHed until we return */
                return (dca_resume(dca));
        case DDI_ATTACH:
                break;
        default:
                return (DDI_FAILURE);
        }

        if (ddi_slaveonly(dip) == DDI_SUCCESS) {
                dca_diperror(dip, "slot does not support PCI bus-master");
                return (DDI_FAILURE);
        }

        if (ddi_intr_hilevel(dip, 0) != 0) {
                dca_diperror(dip, "hilevel interrupts not supported");
                return (DDI_FAILURE);
        }

        if (pci_config_setup(dip, &pci) != DDI_SUCCESS) {
                dca_diperror(dip, "unable to setup PCI config handle");
                return (DDI_FAILURE);
        }

        /* common PCI attributes */
        venid = pci_config_get16(pci, PCI_VENID);
        devid = pci_config_get16(pci, PCI_DEVID);
        revid = pci_config_get8(pci, PCI_REVID);
        subvenid = pci_config_get16(pci, PCI_SUBVENID);
        subsysid = pci_config_get16(pci, PCI_SUBSYSID);

        /*
         * Broadcom-specific timings.
         * We disable these timers/counters since they can cause
         * incorrect false failures when the bus is just a little
         * bit slow, or busy.
         */
        pci_config_put8(pci, PCI_TRDYTO, 0);
        pci_config_put8(pci, PCI_RETRIES, 0);

        /* initialize PCI access settings */
        pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
            PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);

        /* set up our PCI latency timer */
        pci_config_put8(pci, PCI_LATTMR, 0x40);

#if DEBUG
        /* read registers (for debugging) */
        pcicomm = pci_config_get16(pci, PCI_COMM);
        pcistat = pci_config_get16(pci, PCI_STATUS);
        cachelinesz = pci_config_get8(pci, PCI_CACHELINESZ);
        mingnt = pci_config_get8(pci, PCI_MINGNT);
        maxlat = pci_config_get8(pci, PCI_MAXLAT);
        lattmr = pci_config_get8(pci, PCI_LATTMR);
#endif

        pci_config_teardown(&pci);

        if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) {
                dca_diperror(dip, "unable to get iblock cookie");
                return (DDI_FAILURE);
        }

        if (ddi_soft_state_zalloc(dca_state, instance) != DDI_SUCCESS) {
                dca_diperror(dip, "unable to allocate soft state");
                return (DDI_FAILURE);
        }

        dca = ddi_get_soft_state(dca_state, instance);
        ASSERT(dca != NULL);
        dca->dca_dip = dip;
        WORKLIST(dca, MCR1)->dwl_prov = 0;
        WORKLIST(dca, MCR2)->dwl_prov = 0;
        /* figure pagesize */
        dca->dca_pagesize = ddi_ptob(dip, 1);

        /*
         * Search for the device in our supported devices table.  This
         * is here for two reasons.  First, we want to ensure that
         * only Sun-qualified (and presumably Sun-labeled) devices can
         * be used with this driver.  Second, some devices have
         * specific differences.  E.g. the 5821 has support for a
         * special mode of RC4, deeper queues, power management, and
         * other changes.  Also, the export versions of some of these
         * chips don't support RC4 or 3DES, so we catch that here.
         *
         * Note that we only look at the upper nibble of the device
         * id, which is used to distinguish export vs. domestic
         * versions of the chip.  (The lower nibble is used for
         * stepping information.)
         */
        for (i = 0; i < (sizeof (dca_devices) / sizeof (dca_device_t)); i++) {
                /*
                 * Try to match the subsystem information first.
                 */
                if (subvenid && (subvenid == dca_devices[i].dd_vendor_id) &&
                    subsysid && (subsysid == dca_devices[i].dd_device_id)) {
                        dca->dca_model = dca_devices[i].dd_model;
                        dca->dca_devid = dca_devices[i].dd_device_id;
                        break;
                }
                /*
                 * Failing that, try the generic vendor and device id.
                 * Even if we find a match, we keep searching anyway,
                 * since we would prefer to find a match based on the
                 * subsystem ids.
                 */
                if ((venid == dca_devices[i].dd_vendor_id) &&
                    (devid == dca_devices[i].dd_device_id)) {
                        dca->dca_model = dca_devices[i].dd_model;
                        dca->dca_devid = dca_devices[i].dd_device_id;
                }
        }
        /* try and handle an unrecognized device */
        if (dca->dca_model == NULL) {
                dca->dca_model = unknowndev;
                dca_error(dca, "device not recognized, not supported");
                DBG(dca, DPCI, "i=%d venid=%x devid=%x rev=%d",
                    i, venid, devid, revid);
        }

        if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, "description",
            dca->dca_model) != DDI_SUCCESS) {
                dca_error(dca, "unable to create description property");
                return (DDI_FAILURE);
        }

        DBG(dca, DPCI, "PCI command=0x%x status=%x cachelinesz=%x",
            pcicomm, pcistat, cachelinesz);
        DBG(dca, DPCI, "mingnt=0x%x maxlat=0x%x lattmr=0x%x",
            mingnt, maxlat, lattmr);

        /*
         * initialize locks, etc.
         */
        (void) mutex_init(&dca->dca_intrlock, NULL, MUTEX_DRIVER, ibc);

        /* use RNGSHA1 by default */
        if (ddi_getprop(DDI_DEV_T_ANY, dip,
            DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rngdirect", 0) == 0) {
                dca->dca_flags |= DCA_RNGSHA1;
        }

        /* initialize FMA */
        dca_fma_init(dca);

        /* initialize some key data structures */
        if (dca_init(dca) != DDI_SUCCESS) {
                goto failed;
        }

        /* initialize kstats */
        dca_ksinit(dca);

        /* setup access to registers */
        if (ddi_regs_map_setup(dip, 1, (caddr_t *)&dca->dca_regs,
            0, 0, &dca_regsattr, &dca->dca_regs_handle) != DDI_SUCCESS) {
                dca_error(dca, "unable to map registers");
                goto failed;
        }

        DBG(dca, DCHATTY, "MCR1 = %x", GETCSR(dca, CSR_MCR1));
        DBG(dca, DCHATTY, "CONTROL = %x", GETCSR(dca, CSR_DMACTL));
        DBG(dca, DCHATTY, "STATUS = %x", GETCSR(dca, CSR_DMASTAT));
        DBG(dca, DCHATTY, "DMAEA = %x", GETCSR(dca, CSR_DMAEA));
        DBG(dca, DCHATTY, "MCR2 = %x", GETCSR(dca, CSR_MCR2));

        /* reset the chip */
        if (dca_reset(dca, 0) < 0) {
                goto failed;
        }

        /* initialize the chip */
        PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
        if (dca_check_acc_handle(dca, dca->dca_regs_handle,
            DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
                goto failed;
        }

        /* add the interrupt */
        if (ddi_add_intr(dip, 0, &dca->dca_icookie, NULL, dca_intr,
            (void *)dca) != DDI_SUCCESS) {
                DBG(dca, DWARN, "ddi_add_intr failed");
                goto failed;
        } else {
                intr_added = 1;
        }

        /* enable interrupts on the device */
        /*
         * XXX: Note, 5820A1 errata indicates that this may clobber
         * bits 24 and 23, which affect the speed of the RNG.  Since
         * we always want to run in full-speed mode, this should be
         * harmless.
         */
        if (dca->dca_devid == 0x5825) {
                /* for 5825 - increase the DMA read size */
                SETBIT(dca, CSR_DMACTL,
                    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
        } else {
                SETBIT(dca, CSR_DMACTL,
                    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
        }
        if (dca_check_acc_handle(dca, dca->dca_regs_handle,
            DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
                goto failed;
        }

        /* register MCR1 with the crypto framework */
        /* Be careful not to exceed 32 chars */
        (void) sprintf(ID, "%s/%d %s",
            ddi_driver_name(dip), ddi_get_instance(dip), IDENT_SYM);
        dca_prov_info1.pi_provider_description = ID;
        dca_prov_info1.pi_provider_dev.pd_hw = dip;
        dca_prov_info1.pi_provider_handle = dca;
        if ((ret = crypto_register_provider(&dca_prov_info1,
            &WORKLIST(dca, MCR1)->dwl_prov)) != CRYPTO_SUCCESS) {
                cmn_err(CE_WARN,
                    "crypto_register_provider() failed (%d) for MCR1", ret);
                goto failed;
        }

        /* register MCR2 with the crypto framework */
        /* Be careful not to exceed 32 chars */
        (void) sprintf(ID, "%s/%d %s",
            ddi_driver_name(dip), ddi_get_instance(dip), IDENT_ASYM);
        dca_prov_info2.pi_provider_description = ID;
        dca_prov_info2.pi_provider_dev.pd_hw = dip;
        dca_prov_info2.pi_provider_handle = dca;
        if ((ret = crypto_register_provider(&dca_prov_info2,
            &WORKLIST(dca, MCR2)->dwl_prov)) != CRYPTO_SUCCESS) {
                cmn_err(CE_WARN,
                    "crypto_register_provider() failed (%d) for MCR2", ret);
                goto failed;
        }

        crypto_prov_notify(WORKLIST(dca, MCR1)->dwl_prov,
            CRYPTO_PROVIDER_READY);
        crypto_prov_notify(WORKLIST(dca, MCR2)->dwl_prov,
            CRYPTO_PROVIDER_READY);

        /* Initialize the local random number pool for this instance */
        if ((ret = dca_random_init(dca)) != CRYPTO_SUCCESS) {
                goto failed;
        }

        mutex_enter(&dca->dca_intrlock);
        dca->dca_jobtid = timeout(dca_jobtimeout, (void *)dca,
            drv_usectohz(SECOND));
        mutex_exit(&dca->dca_intrlock);

        ddi_set_driver_private(dip, (caddr_t)dca);

        ddi_report_dev(dip);

        if (ddi_get_devstate(dca->dca_dip) != DDI_DEVSTATE_UP) {
                ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_RESTORED);
        }

        return (DDI_SUCCESS);

failed:
        /* unregister from the crypto framework */
        if (WORKLIST(dca, MCR1)->dwl_prov != 0) {
                (void) crypto_unregister_provider(
                    WORKLIST(dca, MCR1)->dwl_prov);
        }
        if (WORKLIST(dca, MCR2)->dwl_prov != 0) {
                (void) crypto_unregister_provider(
                    WORKLIST(dca, MCR2)->dwl_prov);
        }
        if (intr_added) {
                CLRBIT(dca, CSR_DMACTL,
                    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
                /* unregister intr handler */
                ddi_remove_intr(dip, 0, dca->dca_icookie);
        }
        if (dca->dca_regs_handle) {
                ddi_regs_map_free(&dca->dca_regs_handle);
        }
        if (dca->dca_intrstats) {
                kstat_delete(dca->dca_intrstats);
        }
        if (dca->dca_ksp) {
                kstat_delete(dca->dca_ksp);
        }
        dca_uninit(dca);

        /* finalize FMA */
        dca_fma_fini(dca);

        mutex_destroy(&dca->dca_intrlock);
        ddi_soft_state_free(dca_state, instance);
        return (DDI_FAILURE);

}

int
dca_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        int             instance;
        dca_t           *dca;
        timeout_id_t    tid;

        instance = ddi_get_instance(dip);

        DBG(NULL, DMOD, "dca: in dca_detach() for %d", instance);

        switch (cmd) {
        case DDI_SUSPEND:
                if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
                        dca_diperror(dip, "no soft state in detach");
                        return (DDI_FAILURE);
                }
                /* assumption: we won't be DDI_DETACHed until we return */
                return (dca_suspend(dca));

        case DDI_DETACH:
                break;
        default:
                return (DDI_FAILURE);
        }

        if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
                dca_diperror(dip, "no soft state in detach");
                return (DDI_FAILURE);
        }

        /*
         * Unregister from kCF.
         * This needs to be done at the beginning of detach.
         */
        if (WORKLIST(dca, MCR1)->dwl_prov != 0) {
                if (crypto_unregister_provider(
                    WORKLIST(dca, MCR1)->dwl_prov) != CRYPTO_SUCCESS) {
                        dca_error(dca, "unable to unregister MCR1 from kcf");
                        return (DDI_FAILURE);
                }
        }

        if (WORKLIST(dca, MCR2)->dwl_prov != 0) {
                if (crypto_unregister_provider(
                    WORKLIST(dca, MCR2)->dwl_prov) != CRYPTO_SUCCESS) {
                        dca_error(dca, "unable to unregister MCR2 from kcf");
                        return (DDI_FAILURE);
                }
        }

        /*
         * Cleanup the private context list. Once the
         * crypto_unregister_provider returns, it is safe to do so.
         */
        dca_free_context_list(dca);

        /* Cleanup the local random number pool */
        dca_random_fini(dca);

        /* send any jobs in the waitq back to kCF */
        dca_rejectjobs(dca);

        /* untimeout the timeouts */
        mutex_enter(&dca->dca_intrlock);
        tid = dca->dca_jobtid;
        dca->dca_jobtid = 0;
        mutex_exit(&dca->dca_intrlock);
        if (tid) {
                (void) untimeout(tid);
        }

        /* disable device interrupts */
        CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);

        /* unregister interrupt handlers */
        ddi_remove_intr(dip, 0, dca->dca_icookie);

        /* release our regs handle */
        ddi_regs_map_free(&dca->dca_regs_handle);

        /* toss out kstats */
        if (dca->dca_intrstats) {
                kstat_delete(dca->dca_intrstats);
        }
        if (dca->dca_ksp) {
                kstat_delete(dca->dca_ksp);
        }

        mutex_destroy(&dca->dca_intrlock);
        dca_uninit(dca);

        /* finalize FMA */
        dca_fma_fini(dca);

        ddi_soft_state_free(dca_state, instance);

        return (DDI_SUCCESS);
}

int
dca_resume(dca_t *dca)
{
        ddi_acc_handle_t        pci;

        if (pci_config_setup(dca->dca_dip, &pci) != DDI_SUCCESS) {
                dca_error(dca, "unable to setup PCI config handle");
                return (DDI_FAILURE);
        }

        /*
         * Reprogram registers in PCI configuration space.
         */

        /* Broadcom-specific timers -- we disable them. */
        pci_config_put8(pci, PCI_TRDYTO, 0);
        pci_config_put8(pci, PCI_RETRIES, 0);

        /* initialize PCI access settings */
        pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
            PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);

        /* set up our PCI latency timer */
        pci_config_put8(pci, PCI_LATTMR, 0x40);

        pci_config_teardown(&pci);

        if (dca_reset(dca, 0) < 0) {
                dca_error(dca, "unable to reset device during resume");
                return (DDI_FAILURE);
        }

        /*
         * Now restore the card-specific CSRs.
         */

        /* restore endianness settings */
        PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
        if (dca_check_acc_handle(dca, dca->dca_regs_handle,
            DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
                return (DDI_FAILURE);

        /* restore interrupt enables */
        if (dca->dca_devid == 0x5825) {
                /* for 5825 set 256 byte read size to improve performance */
                SETBIT(dca, CSR_DMACTL,
                    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
        } else {
                SETBIT(dca, CSR_DMACTL,
                    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
        }
        if (dca_check_acc_handle(dca, dca->dca_regs_handle,
            DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
                return (DDI_FAILURE);

        /* resume scheduling jobs on the device */
        dca_undrain(dca);

        return (DDI_SUCCESS);
}

int
dca_suspend(dca_t *dca)
{
        if ((dca_drain(dca)) != 0) {
                return (DDI_FAILURE);
        }
        if (dca_reset(dca, 0) < 0) {
                dca_error(dca, "unable to reset device during suspend");
                return (DDI_FAILURE);
        }
        return (DDI_SUCCESS);
}

/*
 * Hardware access stuff.
 */
int
dca_reset(dca_t *dca, int failreset)
{
        int i;

        if (dca->dca_regs_handle == NULL) {
                return (-1);
        }

        PUTCSR(dca, CSR_DMACTL, DMACTL_RESET);
        if (!failreset) {
                if (dca_check_acc_handle(dca, dca->dca_regs_handle,
                    DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
                        return (-1);
        }

        /* now wait for a reset */
        for (i = 1; i < 100; i++) {
                uint32_t        dmactl;
                drv_usecwait(100);
                dmactl = GETCSR(dca, CSR_DMACTL);
                if (!failreset) {
                        if (dca_check_acc_handle(dca, dca->dca_regs_handle,
                            DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
                                return (-1);
                }
                if ((dmactl & DMACTL_RESET) == 0) {
                        DBG(dca, DCHATTY, "reset in %d usec", i * 100);
                        return (0);
                }
        }
        if (!failreset) {
                dca_failure(dca, DDI_DEVICE_FAULT,
                    DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
                    "timeout waiting for reset after %d usec", i * 100);
        }
        return (-1);
}

int
dca_initworklist(dca_t *dca, dca_worklist_t *wlp)
{
        int     i;
        int     reqprealloc = wlp->dwl_hiwater + (MAXWORK * MAXREQSPERMCR);

        /*
         * Set up work queue.
         */
        mutex_init(&wlp->dwl_lock, NULL, MUTEX_DRIVER, dca->dca_icookie);
        mutex_init(&wlp->dwl_freereqslock, NULL, MUTEX_DRIVER,
            dca->dca_icookie);
        mutex_init(&wlp->dwl_freelock, NULL, MUTEX_DRIVER, dca->dca_icookie);
        cv_init(&wlp->dwl_cv, NULL, CV_DRIVER, NULL);

        mutex_enter(&wlp->dwl_lock);

        dca_initq(&wlp->dwl_freereqs);
        dca_initq(&wlp->dwl_waitq);
        dca_initq(&wlp->dwl_freework);
        dca_initq(&wlp->dwl_runq);

        for (i = 0; i < MAXWORK; i++) {
                dca_work_t              *workp;

                if ((workp = dca_newwork(dca)) == NULL) {
                        dca_error(dca, "unable to allocate work");
                        mutex_exit(&wlp->dwl_lock);
                        return (DDI_FAILURE);
                }
                workp->dw_wlp = wlp;
                dca_freework(workp);
        }
        mutex_exit(&wlp->dwl_lock);

        for (i = 0; i < reqprealloc; i++) {
                dca_request_t *reqp;

                if ((reqp = dca_newreq(dca)) == NULL) {
                        dca_error(dca, "unable to allocate request");
                        return (DDI_FAILURE);
                }
                reqp->dr_dca = dca;
                reqp->dr_wlp = wlp;
                dca_freereq(reqp);
        }
        return (DDI_SUCCESS);
}

int
dca_init(dca_t *dca)
{
        dca_worklist_t          *wlp;

        /* Initialize the private context list and the corresponding lock. */
        mutex_init(&dca->dca_ctx_list_lock, NULL, MUTEX_DRIVER, NULL);
        dca_initq(&dca->dca_ctx_list);

        /*
         * MCR1 algorithms.
         */
        wlp = WORKLIST(dca, MCR1);
        (void) sprintf(wlp->dwl_name, "dca%d:mcr1",
            ddi_get_instance(dca->dca_dip));
        wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
            dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
            "mcr1_lowater", MCR1LOWATER);
        wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
            dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
            "mcr1_hiwater", MCR1HIWATER);
        wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
            dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
            "mcr1_maxreqs", MCR1MAXREQS), MAXREQSPERMCR);
        wlp->dwl_dca = dca;
        wlp->dwl_mcr = MCR1;
        if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /*
         * MCR2 algorithms.
         */
        wlp = WORKLIST(dca, MCR2);
        (void) sprintf(wlp->dwl_name, "dca%d:mcr2",
            ddi_get_instance(dca->dca_dip));
        wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
            dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
            "mcr2_lowater", MCR2LOWATER);
        wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
            dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
            "mcr2_hiwater", MCR2HIWATER);
        wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
            dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
            "mcr2_maxreqs", MCR2MAXREQS), MAXREQSPERMCR);
        wlp->dwl_dca = dca;
        wlp->dwl_mcr = MCR2;
        if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }
        return (DDI_SUCCESS);
}

/*
 * Uninitialize worklists.  This routine should only be called when no
 * active jobs (hence DMA mappings) exist.  One way to ensure this is
 * to unregister from kCF before calling this routine.  (This is done
 * e.g. in detach(9e).)
 */
void
dca_uninit(dca_t *dca)
{
        int     mcr;

        mutex_destroy(&dca->dca_ctx_list_lock);

        for (mcr = MCR1; mcr <= MCR2; mcr++) {
                dca_worklist_t  *wlp = WORKLIST(dca, mcr);
                dca_work_t      *workp;
                dca_request_t   *reqp;

                if (dca->dca_regs_handle == NULL) {
                        continue;
                }

                mutex_enter(&wlp->dwl_lock);
                while ((workp = dca_getwork(dca, mcr)) != NULL) {
                        dca_destroywork(workp);
                }
                mutex_exit(&wlp->dwl_lock);
                while ((reqp = dca_getreq(dca, mcr, 0)) != NULL) {
                        dca_destroyreq(reqp);
                }

                mutex_destroy(&wlp->dwl_lock);
                mutex_destroy(&wlp->dwl_freereqslock);
                mutex_destroy(&wlp->dwl_freelock);
                cv_destroy(&wlp->dwl_cv);
                wlp->dwl_prov = 0;
        }
}

static void
dca_enlist2(dca_listnode_t *q, dca_listnode_t *node, kmutex_t *lock)
{
        if (!q || !node)
                return;

        mutex_enter(lock);
        node->dl_next2 = q;
        node->dl_prev2 = q->dl_prev2;
        node->dl_next2->dl_prev2 = node;
        node->dl_prev2->dl_next2 = node;
        mutex_exit(lock);
}

static void
dca_rmlist2(dca_listnode_t *node, kmutex_t *lock)
{
        if (!node)
                return;

        mutex_enter(lock);
        node->dl_next2->dl_prev2 = node->dl_prev2;
        node->dl_prev2->dl_next2 = node->dl_next2;
        node->dl_next2 = NULL;
        node->dl_prev2 = NULL;
        mutex_exit(lock);
}

static dca_listnode_t *
dca_delist2(dca_listnode_t *q, kmutex_t *lock)
{
        dca_listnode_t *node;

        mutex_enter(lock);
        if ((node = q->dl_next2) == q) {
                mutex_exit(lock);
                return (NULL);
        }

        node->dl_next2->dl_prev2 = node->dl_prev2;
        node->dl_prev2->dl_next2 = node->dl_next2;
        node->dl_next2 = NULL;
        node->dl_prev2 = NULL;
        mutex_exit(lock);

        return (node);
}

void
dca_initq(dca_listnode_t *q)
{
        q->dl_next = q;
        q->dl_prev = q;
        q->dl_next2 = q;
        q->dl_prev2 = q;
}

void
dca_enqueue(dca_listnode_t *q, dca_listnode_t *node)
{
        /*
         * Enqueue submits at the "tail" of the list, i.e. just
         * behind the sentinel.
         */
        node->dl_next = q;
        node->dl_prev = q->dl_prev;
        node->dl_next->dl_prev = node;
        node->dl_prev->dl_next = node;
}

void
dca_rmqueue(dca_listnode_t *node)
{
        node->dl_next->dl_prev = node->dl_prev;
        node->dl_prev->dl_next = node->dl_next;
        node->dl_next = NULL;
        node->dl_prev = NULL;
}

dca_listnode_t *
dca_dequeue(dca_listnode_t *q)
{
        dca_listnode_t *node;
        /*
         * Dequeue takes from the "head" of the list, i.e. just after
         * the sentinel.
         */
        if ((node = q->dl_next) == q) {
                /* queue is empty */
                return (NULL);
        }
        dca_rmqueue(node);
        return (node);
}

/* this is the opposite of dequeue, it takes things off in LIFO order */
dca_listnode_t *
dca_unqueue(dca_listnode_t *q)
{
        dca_listnode_t *node;
        /*
         * unqueue takes from the "tail" of the list, i.e. just before
         * the sentinel.
         */
        if ((node = q->dl_prev) == q) {
                /* queue is empty */
                return (NULL);
        }
        dca_rmqueue(node);
        return (node);
}

dca_listnode_t *
dca_peekqueue(dca_listnode_t *q)
{
        dca_listnode_t *node;

        if ((node = q->dl_next) == q) {
                return (NULL);
        } else {
                return (node);
        }
}

/*
 * Interrupt service routine.
 */
uint_t
dca_intr(char *arg)
{
        dca_t           *dca = (dca_t *)arg;
        uint32_t        status;

        mutex_enter(&dca->dca_intrlock);
        status = GETCSR(dca, CSR_DMASTAT);
        PUTCSR(dca, CSR_DMASTAT, status & DMASTAT_INTERRUPTS);
        if (dca_check_acc_handle(dca, dca->dca_regs_handle,
            DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
                mutex_exit(&dca->dca_intrlock);
                return ((uint_t)DDI_FAILURE);
        }

        DBG(dca, DINTR, "interrupted, status = 0x%x!", status);

        if ((status & DMASTAT_INTERRUPTS) == 0) {
                /* increment spurious interrupt kstat */
                if (dca->dca_intrstats) {
                        KIOIP(dca)->intrs[KSTAT_INTR_SPURIOUS]++;
                }
                mutex_exit(&dca->dca_intrlock);
                return (DDI_INTR_UNCLAIMED);
        }

        if (dca->dca_intrstats) {
                KIOIP(dca)->intrs[KSTAT_INTR_HARD]++;
        }
        if (status & DMASTAT_MCR1INT) {
                DBG(dca, DINTR, "MCR1 interrupted");
                mutex_enter(&(WORKLIST(dca, MCR1)->dwl_lock));
                dca_schedule(dca, MCR1);
                dca_reclaim(dca, MCR1);
                mutex_exit(&(WORKLIST(dca, MCR1)->dwl_lock));
        }

        if (status & DMASTAT_MCR2INT) {
                DBG(dca, DINTR, "MCR2 interrupted");
                mutex_enter(&(WORKLIST(dca, MCR2)->dwl_lock));
                dca_schedule(dca, MCR2);
                dca_reclaim(dca, MCR2);
                mutex_exit(&(WORKLIST(dca, MCR2)->dwl_lock));
        }

        if (status & DMASTAT_ERRINT) {
                uint32_t        erraddr;
                erraddr = GETCSR(dca, CSR_DMAEA);
                mutex_exit(&dca->dca_intrlock);

                /*
                 * bit 1 of the error address indicates failure during
                 * read if set, during write otherwise.
                 */
                dca_failure(dca, DDI_DEVICE_FAULT,
                    DCA_FM_ECLASS_HW_DEVICE, dca_ena(0), CRYPTO_DEVICE_ERROR,
                    "DMA master access error %s address 0x%x",
                    erraddr & 0x1 ? "reading" : "writing", erraddr & ~1);
                return (DDI_INTR_CLAIMED);
        }

        mutex_exit(&dca->dca_intrlock);

        return (DDI_INTR_CLAIMED);
}

/*
 * Reverse a string of bytes from s1 into s2.  The reversal happens
 * from the tail of s1.  If len1 < len2, then null bytes will be
 * padded to the end of s2.  If len2 < len1, then (presumably null)
 * bytes will be dropped from the start of s1.
 *
 * The rationale here is that when s1 (source) is shorter, then we
 * are reversing from big-endian ordering, into device ordering, and
 * want to add some extra nulls to the tail (MSB) side of the device.
 *
 * Similarly, when s2 (dest) is shorter, then we are truncating what
 * are presumably null MSB bits from the device.
 *
 * There is an expectation when reversing from the device back into
 * big-endian, that the number of bytes to reverse and the target size
 * will match, and no truncation or padding occurs.
 */
void
dca_reverse(void *s1, void *s2, int len1, int len2)
{
        caddr_t src, dst;

        if (len1 == 0) {
                if (len2) {
                        bzero(s2, len2);
                }
                return;
        }
        src = (caddr_t)s1 + len1 - 1;
        dst = s2;
        while ((src >= (caddr_t)s1) && (len2)) {
                *dst++ = *src--;
                len2--;
        }
        while (len2 > 0) {
                *dst++ = 0;
                len2--;
        }
}

uint16_t
dca_padfull(int num)
{
        if (num <= 512) {
                return (BITS2BYTES(512));
        }
        if (num <= 768) {
                return (BITS2BYTES(768));
        }
        if (num <= 1024) {
                return (BITS2BYTES(1024));
        }
        if (num <= 1536) {
                return (BITS2BYTES(1536));
        }
        if (num <= 2048) {
                return (BITS2BYTES(2048));
        }
        return (0);
}

uint16_t
dca_padhalf(int num)
{
        if (num <= 256) {
                return (BITS2BYTES(256));
        }
        if (num <= 384) {
                return (BITS2BYTES(384));
        }
        if (num <= 512) {
                return (BITS2BYTES(512));
        }
        if (num <= 768) {
                return (BITS2BYTES(768));
        }
        if (num <= 1024) {
                return (BITS2BYTES(1024));
        }
        return (0);
}

dca_work_t *
dca_newwork(dca_t *dca)
{
        dca_work_t              *workp;
        size_t                  size;
        ddi_dma_cookie_t        c;
        unsigned                nc;
        int                     rv;

        workp = kmem_zalloc(sizeof (dca_work_t), KM_SLEEP);

        rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
            DDI_DMA_SLEEP, NULL, &workp->dw_mcr_dmah);
        if (rv != 0) {
                dca_error(dca, "unable to alloc MCR DMA handle");
                dca_destroywork(workp);
                return (NULL);
        }

        rv = ddi_dma_mem_alloc(workp->dw_mcr_dmah,
            ROUNDUP(MCR_SIZE, dca->dca_pagesize),
            &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
            &workp->dw_mcr_kaddr, &size, &workp->dw_mcr_acch);
        if (rv != 0) {
                dca_error(dca, "unable to alloc MCR DMA memory");
                dca_destroywork(workp);
                return (NULL);
        }

        rv = ddi_dma_addr_bind_handle(workp->dw_mcr_dmah, NULL,
            workp->dw_mcr_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_RDWR,
            DDI_DMA_SLEEP, NULL, &c, &nc);
        if (rv != DDI_DMA_MAPPED) {
                dca_error(dca, "unable to map MCR DMA memory");
                dca_destroywork(workp);
                return (NULL);
        }

        workp->dw_mcr_paddr = c.dmac_address;
        return (workp);
}

void
dca_destroywork(dca_work_t *workp)
{
        if (workp->dw_mcr_paddr) {
                (void) ddi_dma_unbind_handle(workp->dw_mcr_dmah);
        }
        if (workp->dw_mcr_acch) {
                ddi_dma_mem_free(&workp->dw_mcr_acch);
        }
        if (workp->dw_mcr_dmah) {
                ddi_dma_free_handle(&workp->dw_mcr_dmah);
        }
        kmem_free(workp, sizeof (dca_work_t));
}

dca_request_t *
dca_newreq(dca_t *dca)
{
        dca_request_t           *reqp;
        size_t                  size;
        ddi_dma_cookie_t        c;
        unsigned                nc;
        int                     rv;
        int                     n_chain = 0;

        size = (DESC_SIZE * MAXFRAGS) + CTX_MAXLENGTH;

        reqp = kmem_zalloc(sizeof (dca_request_t), KM_SLEEP);

        reqp->dr_dca = dca;

        /*
         * Setup the DMA region for the context and descriptors.
         */
        rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP,
            NULL, &reqp->dr_ctx_dmah);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "failure allocating request DMA handle");
                dca_destroyreq(reqp);
                return (NULL);
        }

        /* for driver hardening, allocate in whole pages */
        rv = ddi_dma_mem_alloc(reqp->dr_ctx_dmah,
            ROUNDUP(size, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_kaddr, &size,
            &reqp->dr_ctx_acch);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "unable to alloc request DMA memory");
                dca_destroyreq(reqp);
                return (NULL);
        }

        rv = ddi_dma_addr_bind_handle(reqp->dr_ctx_dmah, NULL,
            reqp->dr_ctx_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_WRITE,
            DDI_DMA_SLEEP, 0, &c, &nc);
        if (rv != DDI_DMA_MAPPED) {
                dca_error(dca, "failed binding request DMA handle");
                dca_destroyreq(reqp);
                return (NULL);
        }
        reqp->dr_ctx_paddr = c.dmac_address;

        reqp->dr_dma_size = size;

        /*
         * Set up the dma for our scratch/shared buffers.
         */
        rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
            DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_dmah);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "failure allocating ibuf DMA handle");
                dca_destroyreq(reqp);
                return (NULL);
        }
        rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
            DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_dmah);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "failure allocating obuf DMA handle");
                dca_destroyreq(reqp);
                return (NULL);
        }

        rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
            DDI_DMA_SLEEP, NULL, &reqp->dr_chain_in_dmah);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "failure allocating chain_in DMA handle");
                dca_destroyreq(reqp);
                return (NULL);
        }

        rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
            DDI_DMA_SLEEP, NULL, &reqp->dr_chain_out_dmah);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "failure allocating chain_out DMA handle");
                dca_destroyreq(reqp);
                return (NULL);
        }

        /*
         * for driver hardening, allocate in whole pages.
         */
        size = ROUNDUP(MAXPACKET, dca->dca_pagesize);
        /*
         * We could kmem_alloc for Sparc too. However, it gives worse
         * performance when transferring more than one page data. For example,
         * using 4 threads and 12032 byte data and 3DES on 900MHZ Sparc system,
         * kmem_alloc uses 80% CPU and ddi_dma_mem_alloc uses 50% CPU for
         * the same throughput.
         */
        rv = ddi_dma_mem_alloc(reqp->dr_ibuf_dmah,
            size, &dca_bufattr,
            DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_kaddr,
            &size, &reqp->dr_ibuf_acch);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "unable to alloc request DMA memory");
                dca_destroyreq(reqp);
                return (NULL);
        }

        rv = ddi_dma_mem_alloc(reqp->dr_obuf_dmah,
            size, &dca_bufattr,
            DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_kaddr,
            &size, &reqp->dr_obuf_acch);
        if (rv != DDI_SUCCESS) {
                dca_error(dca, "unable to alloc request DMA memory");
                dca_destroyreq(reqp);
                return (NULL);
        }

        /* Skip the used portion in the context page */
        reqp->dr_offset = CTX_MAXLENGTH;
        if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
            reqp->dr_ibuf_kaddr, reqp->dr_ibuf_dmah,
            DDI_DMA_WRITE | DDI_DMA_STREAMING,
            &reqp->dr_ibuf_head, &n_chain)) != DDI_SUCCESS) {
                (void) dca_destroyreq(reqp);
                return (NULL);
        }
        reqp->dr_ibuf_paddr = reqp->dr_ibuf_head.dc_buffer_paddr;
        /* Skip the space used by the input buffer */
        reqp->dr_offset += DESC_SIZE * n_chain;

        if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
            reqp->dr_obuf_kaddr, reqp->dr_obuf_dmah,
            DDI_DMA_READ | DDI_DMA_STREAMING,
            &reqp->dr_obuf_head, &n_chain)) != DDI_SUCCESS) {
                (void) dca_destroyreq(reqp);
                return (NULL);
        }
        reqp->dr_obuf_paddr = reqp->dr_obuf_head.dc_buffer_paddr;
        /* Skip the space used by the output buffer */
        reqp->dr_offset += DESC_SIZE * n_chain;

        DBG(dca, DCHATTY, "CTX is 0x%p, phys 0x%x, len %d",
            reqp->dr_ctx_kaddr, reqp->dr_ctx_paddr, CTX_MAXLENGTH);
        return (reqp);
}

void
dca_destroyreq(dca_request_t *reqp)
{

        /*
         * Clean up DMA for the context structure.
         */
        if (reqp->dr_ctx_paddr) {
                (void) ddi_dma_unbind_handle(reqp->dr_ctx_dmah);
        }

        if (reqp->dr_ctx_acch) {
                ddi_dma_mem_free(&reqp->dr_ctx_acch);
        }

        if (reqp->dr_ctx_dmah) {
                ddi_dma_free_handle(&reqp->dr_ctx_dmah);
        }

        /*
         * Clean up DMA for the scratch buffer.
         */
        if (reqp->dr_ibuf_paddr) {
                (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah);
        }
        if (reqp->dr_obuf_paddr) {
                (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah);
        }

        if (reqp->dr_ibuf_acch) {
                ddi_dma_mem_free(&reqp->dr_ibuf_acch);
        }
        if (reqp->dr_obuf_acch) {
                ddi_dma_mem_free(&reqp->dr_obuf_acch);
        }

        if (reqp->dr_ibuf_dmah) {
                ddi_dma_free_handle(&reqp->dr_ibuf_dmah);
        }
        if (reqp->dr_obuf_dmah) {
                ddi_dma_free_handle(&reqp->dr_obuf_dmah);
        }
        /*
         * These two DMA handles should have been unbinded in
         * dca_unbindchains() function
         */
        if (reqp->dr_chain_in_dmah) {
                ddi_dma_free_handle(&reqp->dr_chain_in_dmah);
        }
        if (reqp->dr_chain_out_dmah) {
                ddi_dma_free_handle(&reqp->dr_chain_out_dmah);
        }

        kmem_free(reqp, sizeof (dca_request_t));
}

dca_work_t *
dca_getwork(dca_t *dca, int mcr)
{
        dca_worklist_t  *wlp = WORKLIST(dca, mcr);
        dca_work_t      *workp;

        mutex_enter(&wlp->dwl_freelock);
        workp = (dca_work_t *)dca_dequeue(&wlp->dwl_freework);
        mutex_exit(&wlp->dwl_freelock);
        if (workp) {
                int     nreqs;
                bzero(workp->dw_mcr_kaddr, 8);

                /* clear out old requests */
                for (nreqs = 0; nreqs < MAXREQSPERMCR; nreqs++) {
                        workp->dw_reqs[nreqs] = NULL;
                }
        }
        return (workp);
}

void
dca_freework(dca_work_t *workp)
{
        mutex_enter(&workp->dw_wlp->dwl_freelock);
        dca_enqueue(&workp->dw_wlp->dwl_freework, (dca_listnode_t *)workp);
        mutex_exit(&workp->dw_wlp->dwl_freelock);
}

dca_request_t *
dca_getreq(dca_t *dca, int mcr, int tryhard)
{
        dca_worklist_t  *wlp = WORKLIST(dca, mcr);
        dca_request_t   *reqp;

        mutex_enter(&wlp->dwl_freereqslock);
        reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_freereqs);
        mutex_exit(&wlp->dwl_freereqslock);
        if (reqp) {
                reqp->dr_flags = 0;
                reqp->dr_callback = NULL;
        } else if (tryhard) {
                /*
                 * failed to get a free one, try an allocation, the hard way.
                 * XXX: Kstat desired here.
                 */
                if ((reqp = dca_newreq(dca)) != NULL) {
                        reqp->dr_wlp = wlp;
                        reqp->dr_dca = dca;
                        reqp->dr_flags = 0;
                        reqp->dr_callback = NULL;
                }
        }
        return (reqp);
}

void
dca_freereq(dca_request_t *reqp)
{
        reqp->dr_kcf_req = NULL;
        if (!(reqp->dr_flags & DR_NOCACHE)) {
                mutex_enter(&reqp->dr_wlp->dwl_freereqslock);
                dca_enqueue(&reqp->dr_wlp->dwl_freereqs,
                    (dca_listnode_t *)reqp);
                mutex_exit(&reqp->dr_wlp->dwl_freereqslock);
        }
}

/*
 * Binds user buffers to DMA handles dynamically. On Sparc, a user buffer
 * is mapped to a single physical address. On x86, a user buffer is mapped
 * to multiple physical addresses. These physical addresses are chained
 * using the method specified in Broadcom BCM5820 specification.
 */
int
dca_bindchains(dca_request_t *reqp, size_t incnt, size_t outcnt)
{
        int                     rv;
        caddr_t                 kaddr;
        uint_t                  flags;
        int                     n_chain = 0;

        if (reqp->dr_flags & DR_INPLACE) {
                flags = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
        } else {
                flags = DDI_DMA_WRITE | DDI_DMA_STREAMING;
        }

        /* first the input */
        if (incnt) {
                if ((kaddr = dca_bufdaddr(reqp->dr_in)) == NULL) {
                        DBG(NULL, DWARN, "unrecognised crypto data format");
                        return (DDI_FAILURE);
                }
                if ((rv = dca_bindchains_one(reqp, incnt, reqp->dr_offset,
                    kaddr, reqp->dr_chain_in_dmah, flags,
                    &reqp->dr_chain_in_head, &n_chain)) != DDI_SUCCESS) {
                        (void) dca_unbindchains(reqp);
                        return (rv);
                }

                /*
                 * The offset and length are altered by the calling routine
                 * reqp->dr_in->cd_offset += incnt;
                 * reqp->dr_in->cd_length -= incnt;
                 */
                /* Save the first one in the chain for MCR */
                reqp->dr_in_paddr = reqp->dr_chain_in_head.dc_buffer_paddr;
                reqp->dr_in_next = reqp->dr_chain_in_head.dc_next_paddr;
                reqp->dr_in_len = reqp->dr_chain_in_head.dc_buffer_length;
        } else {
                reqp->dr_in_paddr = 0;
                reqp->dr_in_next = 0;
                reqp->dr_in_len = 0;
        }

        if (reqp->dr_flags & DR_INPLACE) {
                reqp->dr_out_paddr = reqp->dr_in_paddr;
                reqp->dr_out_len = reqp->dr_in_len;
                reqp->dr_out_next = reqp->dr_in_next;
                return (DDI_SUCCESS);
        }

        /* then the output */
        if (outcnt) {
                flags = DDI_DMA_READ | DDI_DMA_STREAMING;
                if ((kaddr = dca_bufdaddr_out(reqp->dr_out)) == NULL) {
                        DBG(NULL, DWARN, "unrecognised crypto data format");
                        (void) dca_unbindchains(reqp);
                        return (DDI_FAILURE);
                }
                rv = dca_bindchains_one(reqp, outcnt, reqp->dr_offset +
                    n_chain * DESC_SIZE, kaddr, reqp->dr_chain_out_dmah,
                    flags, &reqp->dr_chain_out_head, &n_chain);
                if (rv != DDI_SUCCESS) {
                        (void) dca_unbindchains(reqp);
                        return (DDI_FAILURE);
                }

                /* Save the first one in the chain for MCR */
                reqp->dr_out_paddr = reqp->dr_chain_out_head.dc_buffer_paddr;
                reqp->dr_out_next = reqp->dr_chain_out_head.dc_next_paddr;
                reqp->dr_out_len = reqp->dr_chain_out_head.dc_buffer_length;
        } else {
                reqp->dr_out_paddr = 0;
                reqp->dr_out_next = 0;
                reqp->dr_out_len = 0;
        }

        return (DDI_SUCCESS);
}

/*
 * Unbind the user buffers from the DMA handles.
 */
int
dca_unbindchains(dca_request_t *reqp)
{
        int rv = DDI_SUCCESS;
        int rv1 = DDI_SUCCESS;

        /* Clear the input chain */
        if (reqp->dr_chain_in_head.dc_buffer_paddr != 0) {
                (void) ddi_dma_unbind_handle(reqp->dr_chain_in_dmah);
                reqp->dr_chain_in_head.dc_buffer_paddr = 0;
        }

        if (reqp->dr_flags & DR_INPLACE) {
                return (rv);
        }

        /* Clear the output chain */
        if (reqp->dr_chain_out_head.dc_buffer_paddr != 0) {
                (void) ddi_dma_unbind_handle(reqp->dr_chain_out_dmah);
                reqp->dr_chain_out_head.dc_buffer_paddr = 0;
        }

        return ((rv != DDI_SUCCESS)? rv : rv1);
}

/*
 * Build either input chain or output chain. It is single-item chain for Sparc,
 * and possible mutiple-item chain for x86.
 */
static int
dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
    caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
    dca_chain_t *head, int *n_chain)
{
        ddi_dma_cookie_t        c;
        uint_t                  nc;
        int                     rv;
        caddr_t                 chain_kaddr_pre;
        caddr_t                 chain_kaddr;
        uint32_t                chain_paddr;
        int                     i;

        /* Advance past the context structure to the starting address */
        chain_paddr = reqp->dr_ctx_paddr + dr_offset;
        chain_kaddr = reqp->dr_ctx_kaddr + dr_offset;

        /*
         * Bind the kernel address to the DMA handle. On x86, the actual
         * buffer is mapped into multiple physical addresses. On Sparc,
         * the actual buffer is mapped into a single address.
         */
        rv = ddi_dma_addr_bind_handle(handle,
            NULL, kaddr, cnt, flags, DDI_DMA_DONTWAIT, NULL, &c, &nc);
        if (rv != DDI_DMA_MAPPED) {
                return (DDI_FAILURE);
        }

        (void) ddi_dma_sync(handle, 0, cnt, DDI_DMA_SYNC_FORDEV);
        if ((rv = dca_check_dma_handle(reqp->dr_dca, handle,
            DCA_FM_ECLASS_NONE)) != DDI_SUCCESS) {
                reqp->destroy = TRUE;
                return (rv);
        }

        *n_chain = nc;

        /* Setup the data buffer chain for DMA transfer */
        chain_kaddr_pre = NULL;
        head->dc_buffer_paddr = 0;
        head->dc_next_paddr = 0;
        head->dc_buffer_length = 0;
        for (i = 0; i < nc; i++) {
                /* PIO */
                PUTDESC32(reqp, chain_kaddr, DESC_BUFADDR, c.dmac_address);
                PUTDESC16(reqp, chain_kaddr, DESC_RSVD, 0);
                PUTDESC16(reqp, chain_kaddr, DESC_LENGTH, c.dmac_size);

                /* Remember the head of the chain */
                if (head->dc_buffer_paddr == 0) {
                        head->dc_buffer_paddr = c.dmac_address;
                        head->dc_buffer_length = c.dmac_size;
                }

                /* Link to the previous one if one exists */
                if (chain_kaddr_pre) {
                        PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT,
                            chain_paddr);
                        if (head->dc_next_paddr == 0)
                                head->dc_next_paddr = chain_paddr;
                }
                chain_kaddr_pre = chain_kaddr;

                /* Maintain pointers */
                chain_paddr += DESC_SIZE;
                chain_kaddr += DESC_SIZE;

                /* Retrieve the next cookie if there is one */
                if (i < nc-1)
                        ddi_dma_nextcookie(handle, &c);
        }

        /* Set the next pointer in the last entry to NULL */
        PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 0);

        return (DDI_SUCCESS);
}

/*
 * Schedule some work.
 */
int
dca_start(dca_t *dca, dca_request_t *reqp, int mcr, int dosched)
{
        dca_worklist_t  *wlp = WORKLIST(dca, mcr);

        mutex_enter(&wlp->dwl_lock);

        DBG(dca, DCHATTY, "req=%p, in=%p, out=%p, ctx=%p, ibuf=%p, obuf=%p",
            reqp, reqp->dr_in, reqp->dr_out, reqp->dr_ctx_kaddr,
            reqp->dr_ibuf_kaddr, reqp->dr_obuf_kaddr);
        DBG(dca, DCHATTY, "ctx paddr = %x, ibuf paddr = %x, obuf paddr = %x",
            reqp->dr_ctx_paddr, reqp->dr_ibuf_paddr, reqp->dr_obuf_paddr);
        /* sync out the entire context and descriptor chains */
        (void) ddi_dma_sync(reqp->dr_ctx_dmah, 0, 0, DDI_DMA_SYNC_FORDEV);
        if (dca_check_dma_handle(dca, reqp->dr_ctx_dmah,
            DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
                reqp->destroy = TRUE;
                mutex_exit(&wlp->dwl_lock);
                return (CRYPTO_DEVICE_ERROR);
        }

        dca_enqueue(&wlp->dwl_waitq, (dca_listnode_t *)reqp);
        wlp->dwl_count++;
        wlp->dwl_lastsubmit = ddi_get_lbolt();
        reqp->dr_wlp = wlp;

        if ((wlp->dwl_count == wlp->dwl_hiwater) && (wlp->dwl_busy == 0)) {
                /* we are fully loaded now, let kCF know */

                wlp->dwl_flowctl++;
                wlp->dwl_busy = 1;

                crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_BUSY);
        }

        if (dosched) {
#ifdef  SCHEDDELAY
                /* possibly wait for more work to arrive */
                if (wlp->dwl_count >= wlp->dwl_reqspermcr) {
                        dca_schedule(dca, mcr);
                } else if (!wlp->dwl_schedtid) {
                        /* wait 1 msec for more work before doing it */
                        wlp->dwl_schedtid = timeout(dca_schedtimeout,
                            (void *)wlp, drv_usectohz(MSEC));
                }
#else
                dca_schedule(dca, mcr);
#endif
        }
        mutex_exit(&wlp->dwl_lock);

        return (CRYPTO_QUEUED);
}

void
dca_schedule(dca_t *dca, int mcr)
{
        dca_worklist_t  *wlp = WORKLIST(dca, mcr);
        int             csr;
        int             full;
        uint32_t        status;

        ASSERT(mutex_owned(&wlp->dwl_lock));
        /*
         * If the card is draining or has an outstanding failure,
         * don't schedule any more work on it right now
         */
        if (wlp->dwl_drain || (dca->dca_flags & DCA_FAILED)) {
                return;
        }

        if (mcr == MCR2) {
                csr = CSR_MCR2;
                full = DMASTAT_MCR2FULL;
        } else {
                csr = CSR_MCR1;
                full = DMASTAT_MCR1FULL;
        }

        for (;;) {
                dca_work_t      *workp;
                uint32_t        offset;
                int             nreqs;

                status = GETCSR(dca, CSR_DMASTAT);
                if (dca_check_acc_handle(dca, dca->dca_regs_handle,
                    DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
                        return;

                if ((status & full) != 0)
                        break;

#ifdef  SCHEDDELAY
                /* if there isn't enough to do, don't bother now */
                if ((wlp->dwl_count < wlp->dwl_reqspermcr) &&
                    (ddi_get_lbolt() < (wlp->dwl_lastsubmit +
                    drv_usectohz(MSEC)))) {
                        /* wait a bit longer... */
                        if (wlp->dwl_schedtid == 0) {
                                wlp->dwl_schedtid = timeout(dca_schedtimeout,
                                    (void *)wlp, drv_usectohz(MSEC));
                        }
                        return;
                }
#endif

                /* grab a work structure */
                workp = dca_getwork(dca, mcr);

                if (workp == NULL) {
                        /*
                         * There must be work ready to be reclaimed,
                         * in this case, since the chip can only hold
                         * less work outstanding than there are total.
                         */
                        dca_reclaim(dca, mcr);
                        continue;
                }

                nreqs = 0;
                offset = MCR_CTXADDR;

                while (nreqs < wlp->dwl_reqspermcr) {
                        dca_request_t   *reqp;

                        reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_waitq);
                        if (reqp == NULL) {
                                /* nothing left to process */
                                break;
                        }
                        /*
                         * Update flow control.
                         */
                        wlp->dwl_count--;
                        if ((wlp->dwl_count == wlp->dwl_lowater) &&
                            (wlp->dwl_busy))  {
                                wlp->dwl_busy = 0;
                                crypto_prov_notify(wlp->dwl_prov,
                                    CRYPTO_PROVIDER_READY);
                        }

                        /*
                         * Context address.
                         */
                        PUTMCR32(workp, offset, reqp->dr_ctx_paddr);
                        offset += 4;

                        /*
                         * Input chain.
                         */
                        /* input buffer address */
                        PUTMCR32(workp, offset, reqp->dr_in_paddr);
                        offset += 4;
                        /* next input buffer entry */
                        PUTMCR32(workp, offset, reqp->dr_in_next);
                        offset += 4;
                        /* input buffer length */
                        PUTMCR16(workp, offset, reqp->dr_in_len);
                        offset += 2;
                        /* zero the reserved field */
                        PUTMCR16(workp, offset, 0);
                        offset += 2;

                        /*
                         * Overall length.
                         */
                        /* reserved field */
                        PUTMCR16(workp, offset, 0);
                        offset += 2;
                        /* total packet length */
                        PUTMCR16(workp, offset, reqp->dr_pkt_length);
                        offset += 2;

                        /*
                         * Output chain.
                         */
                        /* output buffer address */
                        PUTMCR32(workp, offset, reqp->dr_out_paddr);
                        offset += 4;
                        /* next output buffer entry */
                        PUTMCR32(workp, offset, reqp->dr_out_next);
                        offset += 4;
                        /* output buffer length */
                        PUTMCR16(workp, offset, reqp->dr_out_len);
                        offset += 2;
                        /* zero the reserved field */
                        PUTMCR16(workp, offset, 0);
                        offset += 2;

                        /*
                         * Note submission.
                         */
                        workp->dw_reqs[nreqs] = reqp;
                        nreqs++;
                }

                if (nreqs == 0) {
                        /* nothing in the queue! */
                        dca_freework(workp);
                        return;
                }

                wlp->dwl_submit++;

                PUTMCR16(workp, MCR_FLAGS, 0);
                PUTMCR16(workp, MCR_COUNT, nreqs);

                DBG(dca, DCHATTY,
                    "posting work (phys %x, virt 0x%p) (%d reqs) to MCR%d",
                    workp->dw_mcr_paddr, workp->dw_mcr_kaddr,
                    nreqs, mcr);

                workp->dw_lbolt = ddi_get_lbolt();
                /* Make sure MCR is synced out to device. */
                (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 0,
                    DDI_DMA_SYNC_FORDEV);
                if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
                    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
                        dca_destroywork(workp);
                        return;
                }

                PUTCSR(dca, csr, workp->dw_mcr_paddr);
                if (dca_check_acc_handle(dca, dca->dca_regs_handle,
                    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
                        dca_destroywork(workp);
                        return;
                } else {
                        dca_enqueue(&wlp->dwl_runq, (dca_listnode_t *)workp);
                }

                DBG(dca, DCHATTY, "posted");
        }
}

/*
 * Reclaim completed work, called in interrupt context.
 */
void
dca_reclaim(dca_t *dca, int mcr)
{
        dca_worklist_t  *wlp = WORKLIST(dca, mcr);
        dca_work_t      *workp;
        ushort_t        flags;
        int             nreclaimed = 0;
        int             i;

        DBG(dca, DRECLAIM, "worklist = 0x%p (MCR%d)", wlp, mcr);
        ASSERT(mutex_owned(&wlp->dwl_lock));
        /*
         * For each MCR in the submitted (runq), we check to see if
         * it has been processed.  If so, then we note each individual
         * job in the MCR, and and do the completion processing for
         * each of such job.
         */
        for (;;) {

                workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
                if (workp == NULL) {
                        break;
                }

                /* only sync the MCR flags, since that's all we need */
                (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 4,
                    DDI_DMA_SYNC_FORKERNEL);
                if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
                    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
                        dca_rmqueue((dca_listnode_t *)workp);
                        dca_destroywork(workp);
                        return;
                }

                flags = GETMCR16(workp, MCR_FLAGS);
                if ((flags & MCRFLAG_FINISHED) == 0) {
                        /* chip is still working on it */
                        DBG(dca, DRECLAIM,
                            "chip still working on it (MCR%d)", mcr);
                        break;
                }

                /* its really for us, so remove it from the queue */
                dca_rmqueue((dca_listnode_t *)workp);

                /* if we were draining, signal on the cv */
                if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
                        cv_signal(&wlp->dwl_cv);
                }

                /* update statistics, done under the lock */
                for (i = 0; i < wlp->dwl_reqspermcr; i++) {
                        dca_request_t *reqp = workp->dw_reqs[i];
                        if (reqp == NULL) {
                                continue;
                        }
                        if (reqp->dr_byte_stat >= 0) {
                                dca->dca_stats[reqp->dr_byte_stat] +=
                                    reqp->dr_pkt_length;
                        }
                        if (reqp->dr_job_stat >= 0) {
                                dca->dca_stats[reqp->dr_job_stat]++;
                        }
                }
                mutex_exit(&wlp->dwl_lock);

                for (i = 0; i < wlp->dwl_reqspermcr; i++) {
                        dca_request_t *reqp = workp->dw_reqs[i];

                        if (reqp == NULL) {
                                continue;
                        }

                        /* Do the callback. */
                        workp->dw_reqs[i] = NULL;
                        dca_done(reqp, CRYPTO_SUCCESS);

                        nreclaimed++;
                }

                /* now we can release the work */
                dca_freework(workp);

                mutex_enter(&wlp->dwl_lock);
        }
        DBG(dca, DRECLAIM, "reclaimed %d cmds", nreclaimed);
}

int
dca_length(crypto_data_t *cdata)
{
        return (cdata->cd_length);
}

/*
 * This is the callback function called from the interrupt when a kCF job
 * completes.  It does some driver-specific things, and then calls the
 * kCF-provided callback.  Finally, it cleans up the state for the work
 * request and drops the reference count to allow for DR.
 */
void
dca_done(dca_request_t *reqp, int err)
{
        uint64_t        ena = 0;

        /* unbind any chains we were using */
        if (dca_unbindchains(reqp) != DDI_SUCCESS) {
                /* DMA failure */
                ena = dca_ena(ena);
                dca_failure(reqp->dr_dca, DDI_DATAPATH_FAULT,
                    DCA_FM_ECLASS_NONE, ena, CRYPTO_DEVICE_ERROR,
                    "fault on buffer DMA handle");
                if (err == CRYPTO_SUCCESS) {
                        err = CRYPTO_DEVICE_ERROR;
                }
        }

        if (reqp->dr_callback != NULL) {
                reqp->dr_callback(reqp, err);
        } else {
                dca_freereq(reqp);
        }
}

/*
 * Call this when a failure is detected.  It will reset the chip,
 * log a message, alert kCF, and mark jobs in the runq as failed.
 */
/* ARGSUSED */
void
dca_failure(dca_t *dca, ddi_fault_location_t loc, dca_fma_eclass_t index,
    uint64_t ena, int errno, char *mess, ...)
{
        va_list ap;
        char    buf[256];
        int     mcr;
        char    *eclass;
        int     have_mutex;

        va_start(ap, mess);
        (void) vsprintf(buf, mess, ap);
        va_end(ap);

        eclass = dca_fma_eclass_string(dca->dca_model, index);

        if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) &&
            index != DCA_FM_ECLASS_NONE) {
                ddi_fm_ereport_post(dca->dca_dip, eclass, ena,
                    DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
                    FM_EREPORT_VERS0, NULL);

                /* Report the impact of the failure to the DDI. */
                ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_LOST);
        } else {
                /* Just log the error string to the message log */
                dca_error(dca, buf);
        }

        /*
         * Indicate a failure (keeps schedule from running).
         */
        dca->dca_flags |= DCA_FAILED;

        /*
         * Reset the chip.  This should also have as a side effect, the
         * disabling of all interrupts from the device.
         */
        (void) dca_reset(dca, 1);

        /*
         * Report the failure to kCF.
         */
        for (mcr = MCR1; mcr <= MCR2; mcr++) {
                if (WORKLIST(dca, mcr)->dwl_prov) {
                        crypto_prov_notify(WORKLIST(dca, mcr)->dwl_prov,
                            CRYPTO_PROVIDER_FAILED);
                }
        }

        /*
         * Return jobs not sent to hardware back to kCF.
         */
        dca_rejectjobs(dca);

        /*
         * From this point on, no new work should be arriving, and the
         * chip should not be doing any active DMA.
         */

        /*
         * Now find all the work submitted to the device and fail
         * them.
         */
        for (mcr = MCR1; mcr <= MCR2; mcr++) {
                dca_worklist_t  *wlp;
                int             i;

                wlp = WORKLIST(dca, mcr);

                if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
                        continue;
                }
                for (;;) {
                        dca_work_t      *workp;

                        have_mutex = mutex_tryenter(&wlp->dwl_lock);
                        workp = (dca_work_t *)dca_dequeue(&wlp->dwl_runq);
                        if (workp == NULL) {
                                if (have_mutex)
                                        mutex_exit(&wlp->dwl_lock);
                                break;
                        }
                        mutex_exit(&wlp->dwl_lock);

                        /*
                         * Free up requests
                         */
                        for (i = 0; i < wlp->dwl_reqspermcr; i++) {
                                dca_request_t *reqp = workp->dw_reqs[i];
                                if (reqp) {
                                        dca_done(reqp, errno);
                                        workp->dw_reqs[i] = NULL;
                                }
                        }

                        mutex_enter(&wlp->dwl_lock);
                        /*
                         * If waiting to drain, signal on the waiter.
                         */
                        if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
                                cv_signal(&wlp->dwl_cv);
                        }

                        /*
                         * Return the work and request structures to
                         * the free pool.
                         */
                        dca_freework(workp);
                        if (have_mutex)
                                mutex_exit(&wlp->dwl_lock);
                }
        }

}

#ifdef  SCHEDDELAY
/*
 * Reschedule worklist as needed.
 */
void
dca_schedtimeout(void *arg)
{
        dca_worklist_t  *wlp = (dca_worklist_t *)arg;
        mutex_enter(&wlp->dwl_lock);
        wlp->dwl_schedtid = 0;
        dca_schedule(wlp->dwl_dca, wlp->dwl_mcr);
        mutex_exit(&wlp->dwl_lock);
}
#endif

/*
 * Check for stalled jobs.
 */
void
dca_jobtimeout(void *arg)
{
        int             mcr;
        dca_t           *dca = (dca_t *)arg;
        int             hung = 0;

        for (mcr = MCR1; mcr <= MCR2; mcr++) {
                dca_worklist_t  *wlp = WORKLIST(dca, mcr);
                dca_work_t      *workp;
                clock_t         when;

                mutex_enter(&wlp->dwl_lock);
                when = ddi_get_lbolt();

                workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
                if (workp == NULL) {
                        /* nothing sitting in the queue */
                        mutex_exit(&wlp->dwl_lock);
                        continue;
                }

                if ((when - workp->dw_lbolt) < drv_usectohz(STALETIME)) {
                        /* request has been queued for less than STALETIME */
                        mutex_exit(&wlp->dwl_lock);
                        continue;
                }

                /* job has been sitting around for over 1 second, badness */
                DBG(dca, DWARN, "stale job (0x%p) found in MCR%d!", workp,
                    mcr);

                /* put it back in the queue, until we reset the chip */
                hung++;
                mutex_exit(&wlp->dwl_lock);
        }

        if (hung) {
                dca_failure(dca, DDI_DEVICE_FAULT,
                    DCA_FM_ECLASS_HW_TIMEOUT, dca_ena(0), CRYPTO_DEVICE_ERROR,
                    "timeout processing job.)");
        }

        /* reschedule ourself */
        mutex_enter(&dca->dca_intrlock);
        if (dca->dca_jobtid == 0) {
                /* timeout has been canceled, prior to DR */
                mutex_exit(&dca->dca_intrlock);
                return;
        }

        /* check again in 1 second */
        dca->dca_jobtid = timeout(dca_jobtimeout, arg,
            drv_usectohz(SECOND));
        mutex_exit(&dca->dca_intrlock);
}

/*
 * This returns all jobs back to kCF.  It assumes that processing
 * on the worklist has halted.
 */
void
dca_rejectjobs(dca_t *dca)
{
        int mcr;
        int have_mutex;
        for (mcr = MCR1; mcr <= MCR2; mcr++) {
                dca_worklist_t  *wlp = WORKLIST(dca, mcr);
                dca_request_t   *reqp;

                if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
                        continue;
                }
                have_mutex = mutex_tryenter(&wlp->dwl_lock);
                for (;;) {
                        reqp = (dca_request_t *)dca_unqueue(&wlp->dwl_waitq);
                        if (reqp == NULL) {
                                break;
                        }
                        /* update flow control */
                        wlp->dwl_count--;
                        if ((wlp->dwl_count == wlp->dwl_lowater) &&
                            (wlp->dwl_busy))  {
                                wlp->dwl_busy = 0;
                                crypto_prov_notify(wlp->dwl_prov,
                                    CRYPTO_PROVIDER_READY);
                        }
                        mutex_exit(&wlp->dwl_lock);

                        (void) dca_unbindchains(reqp);
                        reqp->dr_callback(reqp, EAGAIN);
                        mutex_enter(&wlp->dwl_lock);
                }
                if (have_mutex)
                        mutex_exit(&wlp->dwl_lock);
        }
}

int
dca_drain(dca_t *dca)
{
        int mcr;
        for (mcr = MCR1; mcr <= MCR2; mcr++) {
#ifdef  SCHEDDELAY
                timeout_id_t    tid;
#endif
                dca_worklist_t *wlp = WORKLIST(dca, mcr);

                mutex_enter(&wlp->dwl_lock);
                wlp->dwl_drain = 1;

                /* give it up to a second to drain from the chip */
                if (!QEMPTY(&wlp->dwl_runq)) {
                        (void) cv_reltimedwait(&wlp->dwl_cv, &wlp->dwl_lock,
                            drv_usectohz(STALETIME), TR_CLOCK_TICK);

                        if (!QEMPTY(&wlp->dwl_runq)) {
                                dca_error(dca, "unable to drain device");
                                mutex_exit(&wlp->dwl_lock);
                                dca_undrain(dca);
                                return (EBUSY);
                        }
                }

#ifdef  SCHEDDELAY
                tid = wlp->dwl_schedtid;
                mutex_exit(&wlp->dwl_lock);

                /*
                 * untimeout outside the lock -- this is safe because we
                 * have set the drain flag, so dca_schedule() will not
                 * reschedule another timeout
                 */
                if (tid) {
                        untimeout(tid);
                }
#else
                mutex_exit(&wlp->dwl_lock);
#endif
        }
        return (0);
}

void
dca_undrain(dca_t *dca)
{
        int     mcr;

        for (mcr = MCR1; mcr <= MCR2; mcr++) {
                dca_worklist_t  *wlp = WORKLIST(dca, mcr);
                mutex_enter(&wlp->dwl_lock);
                wlp->dwl_drain = 0;
                dca_schedule(dca, mcr);
                mutex_exit(&wlp->dwl_lock);
        }
}

/*
 * Duplicate the crypto_data_t structure, but point to the original
 * buffers.
 */
int
dca_dupcrypto(crypto_data_t *input, crypto_data_t *ninput)
{
        ninput->cd_format = input->cd_format;
        ninput->cd_offset = input->cd_offset;
        ninput->cd_length = input->cd_length;
        ninput->cd_miscdata = input->cd_miscdata;

        switch (input->cd_format) {
        case CRYPTO_DATA_RAW:
                ninput->cd_raw.iov_base = input->cd_raw.iov_base;
                ninput->cd_raw.iov_len = input->cd_raw.iov_len;
                break;

        case CRYPTO_DATA_UIO:
                ninput->cd_uio = input->cd_uio;
                break;

        case CRYPTO_DATA_MBLK:
                ninput->cd_mp = input->cd_mp;
                break;

        default:
                DBG(NULL, DWARN,
                    "dca_dupcrypto: unrecognised crypto data format");
                return (CRYPTO_FAILED);
        }

        return (CRYPTO_SUCCESS);
}

/*
 * Performs validation checks on the input and output data structures.
 */
int
dca_verifyio(crypto_data_t *input, crypto_data_t *output)
{
        int     rv = CRYPTO_SUCCESS;

        switch (input->cd_format) {
        case CRYPTO_DATA_RAW:
                break;

        case CRYPTO_DATA_UIO:
                /* we support only kernel buffer */
                if (input->cd_uio->uio_segflg != UIO_SYSSPACE) {
                        DBG(NULL, DWARN, "non kernel input uio buffer");
                        rv = CRYPTO_ARGUMENTS_BAD;
                }
                break;

        case CRYPTO_DATA_MBLK:
                break;

        default:
                DBG(NULL, DWARN, "unrecognised input crypto data format");
                rv = CRYPTO_ARGUMENTS_BAD;
        }

        switch (output->cd_format) {
        case CRYPTO_DATA_RAW:
                break;

        case CRYPTO_DATA_UIO:
                /* we support only kernel buffer */
                if (output->cd_uio->uio_segflg != UIO_SYSSPACE) {
                        DBG(NULL, DWARN, "non kernel output uio buffer");
                        rv = CRYPTO_ARGUMENTS_BAD;
                }
                break;

        case CRYPTO_DATA_MBLK:
                break;

        default:
                DBG(NULL, DWARN, "unrecognised output crypto data format");
                rv = CRYPTO_ARGUMENTS_BAD;
        }

        return (rv);
}

/*
 * data: source crypto_data_t struct
 * off: offset into the source before commencing copy
 * count: the amount of data to copy
 * dest: destination buffer
 */
int
dca_getbufbytes(crypto_data_t *data, size_t off, int count, uchar_t *dest)
{
        int rv = CRYPTO_SUCCESS;
        uio_t *uiop;
        uint_t vec_idx;
        size_t cur_len;
        mblk_t *mp;

        if (count == 0) {
                /* We don't want anything so we're done. */
                return (rv);
        }

        /*
         * Sanity check that we haven't specified a length greater than the
         * offset adjusted size of the buffer.
         */
        if (count > (data->cd_length - off)) {
                return (CRYPTO_DATA_LEN_RANGE);
        }

        /* Add the internal crypto_data offset to the requested offset. */
        off += data->cd_offset;

        switch (data->cd_format) {
        case CRYPTO_DATA_RAW:
                bcopy(data->cd_raw.iov_base + off, dest, count);
                break;

        case CRYPTO_DATA_UIO:
                /*
                 * Jump to the first iovec containing data to be
                 * processed.
                 */
                uiop = data->cd_uio;
                for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
                    off >= uiop->uio_iov[vec_idx].iov_len;
                    off -= uiop->uio_iov[vec_idx++].iov_len)
                        ;
                if (vec_idx == uiop->uio_iovcnt) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now process the iovecs.
                 */
                while (vec_idx < uiop->uio_iovcnt && count > 0) {
                        cur_len = min(uiop->uio_iov[vec_idx].iov_len -
                            off, count);
                        bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
                            cur_len);
                        count -= cur_len;
                        dest += cur_len;
                        vec_idx++;
                        off = 0;
                }

                if (vec_idx == uiop->uio_iovcnt && count > 0) {
                        /*
                         * The end of the specified iovec's was reached but
                         * the length requested could not be processed
                         * (requested to digest more data than it provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        case CRYPTO_DATA_MBLK:
                /*
                 * Jump to the first mblk_t containing data to be processed.
                 */
                for (mp = data->cd_mp; mp != NULL && off >= MBLKL(mp);
                    off -= MBLKL(mp), mp = mp->b_cont)
                        ;
                if (mp == NULL) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now do the processing on the mblk chain.
                 */
                while (mp != NULL && count > 0) {
                        cur_len = min(MBLKL(mp) - off, count);
                        bcopy((char *)(mp->b_rptr + off), dest, cur_len);
                        count -= cur_len;
                        dest += cur_len;
                        mp = mp->b_cont;
                        off = 0;
                }

                if (mp == NULL && count > 0) {
                        /*
                         * The end of the mblk was reached but the length
                         * requested could not be processed, (requested to
                         * digest more data than it provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        default:
                DBG(NULL, DWARN, "unrecognised crypto data format");
                rv = CRYPTO_ARGUMENTS_BAD;
        }
        return (rv);
}


/*
 * Performs the input, output or hard scatter/gather checks on the specified
 * crypto_data_t struct. Returns true if the data is scatter/gather in nature
 * ie fails the test.
 */
int
dca_sgcheck(dca_t *dca, crypto_data_t *data, dca_sg_param_t val)
{
        uio_t *uiop;
        mblk_t *mp;
        int rv = FALSE;

        switch (val) {
        case DCA_SG_CONTIG:
                /*
                 * Check for a contiguous data buffer.
                 */
                switch (data->cd_format) {
                case CRYPTO_DATA_RAW:
                        /* Contiguous in nature */
                        break;

                case CRYPTO_DATA_UIO:
                        if (data->cd_uio->uio_iovcnt > 1)
                                rv = TRUE;
                        break;

                case CRYPTO_DATA_MBLK:
                        mp = data->cd_mp;
                        if (mp->b_cont != NULL)
                                rv = TRUE;
                        break;

                default:
                        DBG(NULL, DWARN, "unrecognised crypto data format");
                }
                break;

        case DCA_SG_WALIGN:
                /*
                 * Check for a contiguous data buffer that is 32-bit word
                 * aligned and is of word multiples in size.
                 */
                switch (data->cd_format) {
                case CRYPTO_DATA_RAW:
                        if ((data->cd_raw.iov_len % sizeof (uint32_t)) ||
                            ((uintptr_t)data->cd_raw.iov_base %
                            sizeof (uint32_t))) {
                                rv = TRUE;
                        }
                        break;

                case CRYPTO_DATA_UIO:
                        uiop = data->cd_uio;
                        if (uiop->uio_iovcnt > 1) {
                                return (TRUE);
                        }
                        /* So there is only one iovec */
                        if ((uiop->uio_iov[0].iov_len % sizeof (uint32_t)) ||
                            ((uintptr_t)uiop->uio_iov[0].iov_base %
                            sizeof (uint32_t))) {
                                rv = TRUE;
                        }
                        break;

                case CRYPTO_DATA_MBLK:
                        mp = data->cd_mp;
                        if (mp->b_cont != NULL) {
                                return (TRUE);
                        }
                        /* So there is only one mblk in the chain */
                        if ((MBLKL(mp) % sizeof (uint32_t)) ||
                            ((uintptr_t)mp->b_rptr % sizeof (uint32_t))) {
                                rv = TRUE;
                        }
                        break;

                default:
                        DBG(NULL, DWARN, "unrecognised crypto data format");
                }
                break;

        case DCA_SG_PALIGN:
                /*
                 * Check that the data buffer is page aligned and is of
                 * page multiples in size.
                 */
                switch (data->cd_format) {
                case CRYPTO_DATA_RAW:
                        if ((data->cd_length % dca->dca_pagesize) ||
                            ((uintptr_t)data->cd_raw.iov_base %
                            dca->dca_pagesize)) {
                                rv = TRUE;
                        }
                        break;

                case CRYPTO_DATA_UIO:
                        uiop = data->cd_uio;
                        if ((uiop->uio_iov[0].iov_len % dca->dca_pagesize) ||
                            ((uintptr_t)uiop->uio_iov[0].iov_base %
                            dca->dca_pagesize)) {
                                rv = TRUE;
                        }
                        break;

                case CRYPTO_DATA_MBLK:
                        mp = data->cd_mp;
                        if ((MBLKL(mp) % dca->dca_pagesize) ||
                            ((uintptr_t)mp->b_rptr % dca->dca_pagesize)) {
                                rv = TRUE;
                        }
                        break;

                default:
                        DBG(NULL, DWARN, "unrecognised crypto data format");
                }
                break;

        default:
                DBG(NULL, DWARN, "unrecognised scatter/gather param type");
        }

        return (rv);
}

/*
 * Increments the cd_offset and decrements the cd_length as the data is
 * gathered from the crypto_data_t struct.
 * The data is reverse-copied into the dest buffer if the flag is true.
 */
int
dca_gather(crypto_data_t *in, char *dest, int count, int reverse)
{
        int     rv = CRYPTO_SUCCESS;
        uint_t  vec_idx;
        uio_t   *uiop;
        off_t   off = in->cd_offset;
        size_t  cur_len;
        mblk_t  *mp;

        switch (in->cd_format) {
        case CRYPTO_DATA_RAW:
                if (count > in->cd_length) {
                        /*
                         * The caller specified a length greater than the
                         * size of the buffer.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                if (reverse)
                        dca_reverse(in->cd_raw.iov_base + off, dest, count,
                            count);
                else
                        bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
                in->cd_offset += count;
                in->cd_length -= count;
                break;

        case CRYPTO_DATA_UIO:
                /*
                 * Jump to the first iovec containing data to be processed.
                 */
                uiop = in->cd_uio;
                for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
                    off >= uiop->uio_iov[vec_idx].iov_len;
                    off -= uiop->uio_iov[vec_idx++].iov_len)
                        ;
                if (vec_idx == uiop->uio_iovcnt) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now process the iovecs.
                 */
                while (vec_idx < uiop->uio_iovcnt && count > 0) {
                        cur_len = min(uiop->uio_iov[vec_idx].iov_len -
                            off, count);
                        count -= cur_len;
                        if (reverse) {
                                /* Fill the dest buffer from the end */
                                dca_reverse(uiop->uio_iov[vec_idx].iov_base +
                                    off, dest+count, cur_len, cur_len);
                        } else {
                                bcopy(uiop->uio_iov[vec_idx].iov_base + off,
                                    dest, cur_len);
                                dest += cur_len;
                        }
                        in->cd_offset += cur_len;
                        in->cd_length -= cur_len;
                        vec_idx++;
                        off = 0;
                }

                if (vec_idx == uiop->uio_iovcnt && count > 0) {
                        /*
                         * The end of the specified iovec's was reached but
                         * the length requested could not be processed
                         * (requested to digest more data than it provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        case CRYPTO_DATA_MBLK:
                /*
                 * Jump to the first mblk_t containing data to be processed.
                 */
                for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
                    off -= MBLKL(mp), mp = mp->b_cont)
                        ;
                if (mp == NULL) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now do the processing on the mblk chain.
                 */
                while (mp != NULL && count > 0) {
                        cur_len = min(MBLKL(mp) - off, count);
                        count -= cur_len;
                        if (reverse) {
                                /* Fill the dest buffer from the end */
                                dca_reverse((char *)(mp->b_rptr + off),
                                    dest+count, cur_len, cur_len);
                        } else {
                                bcopy((char *)(mp->b_rptr + off), dest,
                                    cur_len);
                                dest += cur_len;
                        }
                        in->cd_offset += cur_len;
                        in->cd_length -= cur_len;
                        mp = mp->b_cont;
                        off = 0;
                }

                if (mp == NULL && count > 0) {
                        /*
                         * The end of the mblk was reached but the length
                         * requested could not be processed, (requested to
                         * digest more data than it provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        default:
                DBG(NULL, DWARN, "dca_gather: unrecognised crypto data format");
                rv = CRYPTO_ARGUMENTS_BAD;
        }
        return (rv);
}

/*
 * Increments the cd_offset and decrements the cd_length as the data is
 * gathered from the crypto_data_t struct.
 */
int
dca_resid_gather(crypto_data_t *in, char *resid, int *residlen, char *dest,
    int count)
{
        int     rv = CRYPTO_SUCCESS;
        caddr_t baddr;
        uint_t  vec_idx;
        uio_t   *uiop;
        off_t   off = in->cd_offset;
        size_t  cur_len;
        mblk_t  *mp;

        /* Process the residual first */
        if (*residlen > 0) {
                uint_t  num = min(count, *residlen);
                bcopy(resid, dest, num);
                *residlen -= num;
                if (*residlen > 0) {
                        /*
                         * Requested amount 'count' is less than what's in
                         * the residual, so shuffle any remaining resid to
                         * the front.
                         */
                        baddr = resid + num;
                        bcopy(baddr, resid, *residlen);
                }
                dest += num;
                count -= num;
        }

        /* Now process what's in the crypto_data_t structs */
        switch (in->cd_format) {
        case CRYPTO_DATA_RAW:
                if (count > in->cd_length) {
                        /*
                         * The caller specified a length greater than the
                         * size of the buffer.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
                in->cd_offset += count;
                in->cd_length -= count;
                break;

        case CRYPTO_DATA_UIO:
                /*
                 * Jump to the first iovec containing data to be processed.
                 */
                uiop = in->cd_uio;
                for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
                    off >= uiop->uio_iov[vec_idx].iov_len;
                    off -= uiop->uio_iov[vec_idx++].iov_len)
                        ;
                if (vec_idx == uiop->uio_iovcnt) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now process the iovecs.
                 */
                while (vec_idx < uiop->uio_iovcnt && count > 0) {
                        cur_len = min(uiop->uio_iov[vec_idx].iov_len -
                            off, count);
                        bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
                            cur_len);
                        count -= cur_len;
                        dest += cur_len;
                        in->cd_offset += cur_len;
                        in->cd_length -= cur_len;
                        vec_idx++;
                        off = 0;
                }

                if (vec_idx == uiop->uio_iovcnt && count > 0) {
                        /*
                         * The end of the specified iovec's was reached but
                         * the length requested could not be processed
                         * (requested to digest more data than it provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        case CRYPTO_DATA_MBLK:
                /*
                 * Jump to the first mblk_t containing data to be processed.
                 */
                for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
                    off -= MBLKL(mp), mp = mp->b_cont)
                        ;
                if (mp == NULL) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now do the processing on the mblk chain.
                 */
                while (mp != NULL && count > 0) {
                        cur_len = min(MBLKL(mp) - off, count);
                        bcopy((char *)(mp->b_rptr + off), dest, cur_len);
                        count -= cur_len;
                        dest += cur_len;
                        in->cd_offset += cur_len;
                        in->cd_length -= cur_len;
                        mp = mp->b_cont;
                        off = 0;
                }

                if (mp == NULL && count > 0) {
                        /*
                         * The end of the mblk was reached but the length
                         * requested could not be processed, (requested to
                         * digest more data than it provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        default:
                DBG(NULL, DWARN,
                    "dca_resid_gather: unrecognised crypto data format");
                rv = CRYPTO_ARGUMENTS_BAD;
        }
        return (rv);
}

/*
 * Appends the data to the crypto_data_t struct increasing cd_length.
 * cd_offset is left unchanged.
 * Data is reverse-copied if the flag is TRUE.
 */
int
dca_scatter(const char *src, crypto_data_t *out, int count, int reverse)
{
        int     rv = CRYPTO_SUCCESS;
        off_t   offset = out->cd_offset + out->cd_length;
        uint_t  vec_idx;
        uio_t   *uiop;
        size_t  cur_len;
        mblk_t  *mp;

        switch (out->cd_format) {
        case CRYPTO_DATA_RAW:
                if (out->cd_raw.iov_len - offset < count) {
                        /* Trying to write out more than space available. */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                if (reverse)
                        dca_reverse((void*) src, out->cd_raw.iov_base + offset,
                            count, count);
                else
                        bcopy(src, out->cd_raw.iov_base + offset, count);
                out->cd_length += count;
                break;

        case CRYPTO_DATA_UIO:
                /*
                 * Jump to the first iovec that can be written to.
                 */
                uiop = out->cd_uio;
                for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
                    offset >= uiop->uio_iov[vec_idx].iov_len;
                    offset -= uiop->uio_iov[vec_idx++].iov_len)
                        ;
                if (vec_idx == uiop->uio_iovcnt) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now process the iovecs.
                 */
                while (vec_idx < uiop->uio_iovcnt && count > 0) {
                        cur_len = min(uiop->uio_iov[vec_idx].iov_len -
                            offset, count);
                        count -= cur_len;
                        if (reverse) {
                                dca_reverse((void*) (src+count),
                                    uiop->uio_iov[vec_idx].iov_base +
                                    offset, cur_len, cur_len);
                        } else {
                                bcopy(src, uiop->uio_iov[vec_idx].iov_base +
                                    offset, cur_len);
                                src += cur_len;
                        }
                        out->cd_length += cur_len;
                        vec_idx++;
                        offset = 0;
                }

                if (vec_idx == uiop->uio_iovcnt && count > 0) {
                        /*
                         * The end of the specified iovec's was reached but
                         * the length requested could not be processed
                         * (requested to write more data than space provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        case CRYPTO_DATA_MBLK:
                /*
                 * Jump to the first mblk_t that can be written to.
                 */
                for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp);
                    offset -= MBLKL(mp), mp = mp->b_cont)
                        ;
                if (mp == NULL) {
                        /*
                         * The caller specified an offset that is larger than
                         * the total size of the buffers it provided.
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }

                /*
                 * Now do the processing on the mblk chain.
                 */
                while (mp != NULL && count > 0) {
                        cur_len = min(MBLKL(mp) - offset, count);
                        count -= cur_len;
                        if (reverse) {
                                dca_reverse((void*) (src+count),
                                    (char *)(mp->b_rptr + offset), cur_len,
                                    cur_len);
                        } else {
                                bcopy(src, (char *)(mp->b_rptr + offset),
                                    cur_len);
                                src += cur_len;
                        }
                        out->cd_length += cur_len;
                        mp = mp->b_cont;
                        offset = 0;
                }

                if (mp == NULL && count > 0) {
                        /*
                         * The end of the mblk was reached but the length
                         * requested could not be processed, (requested to
                         * digest more data than it provided).
                         */
                        return (CRYPTO_DATA_LEN_RANGE);
                }
                break;

        default:
                DBG(NULL, DWARN, "unrecognised crypto data format");
                rv = CRYPTO_ARGUMENTS_BAD;
        }
        return (rv);
}

/*
 * Compare two byte arrays in reverse order.
 * Return 0 if they are identical, 1 otherwise.
 */
int
dca_bcmp_reverse(const void *s1, const void *s2, size_t n)
{
        int i;
        caddr_t src, dst;

        if (!n)
                return (0);

        src = ((caddr_t)s1) + n - 1;
        dst = (caddr_t)s2;
        for (i = 0; i < n; i++) {
                if (*src != *dst)
                        return (1);
                src--;
                dst++;
        }

        return (0);
}


/*
 * This calculates the size of a bignum in bits, specifically not counting
 * leading zero bits.  This size calculation must be done *before* any
 * endian reversal takes place (i.e. the numbers are in absolute big-endian
 * order.)
 */
int
dca_bitlen(unsigned char *bignum, int bytelen)
{
        unsigned char   msbyte;
        int             i, j;

        for (i = 0; i < bytelen - 1; i++) {
                if (bignum[i] != 0) {
                        break;
                }
        }
        msbyte = bignum[i];
        for (j = 8; j > 1; j--) {
                if (msbyte & 0x80) {
                        break;
                }
                msbyte <<= 1;
        }
        return ((8 * (bytelen - i - 1)) + j);
}

/*
 * This compares to bignums (in big-endian order).  It ignores leading
 * null bytes.  The result semantics follow bcmp, mempcmp, strcmp, etc.
 */
int
dca_numcmp(caddr_t n1, int n1len, caddr_t n2, int n2len)
{
        while ((n1len > 1) && (*n1 == 0)) {
                n1len--;
                n1++;
        }
        while ((n2len > 1) && (*n2 == 0)) {
                n2len--;
                n2++;
        }
        if (n1len != n2len) {
                return (n1len - n2len);
        }
        while ((n1len > 1) && (*n1 == *n2)) {
                n1++;
                n2++;
                n1len--;
        }
        return ((int)(*(uchar_t *)n1) - (int)(*(uchar_t *)n2));
}

/*
 * Return array of key attributes.
 */
crypto_object_attribute_t *
dca_get_key_attr(crypto_key_t *key)
{
        if ((key->ck_format != CRYPTO_KEY_ATTR_LIST) ||
            (key->ck_count == 0)) {
                return (NULL);
        }

        return (key->ck_attrs);
}

/*
 * If attribute type exists valp points to it's 32-bit value.
 */
int
dca_attr_lookup_uint32(crypto_object_attribute_t *attrp, uint_t atnum,
    uint64_t atype, uint32_t *valp)
{
        crypto_object_attribute_t       *bap;

        bap = dca_find_attribute(attrp, atnum, atype);
        if (bap == NULL) {
                return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
        }

        *valp = *bap->oa_value;

        return (CRYPTO_SUCCESS);
}

/*
 * If attribute type exists data contains the start address of the value,
 * and numelems contains it's length.
 */
int
dca_attr_lookup_uint8_array(crypto_object_attribute_t *attrp, uint_t atnum,
    uint64_t atype, void **data, unsigned int *numelems)
{
        crypto_object_attribute_t       *bap;

        bap = dca_find_attribute(attrp, atnum, atype);
        if (bap == NULL) {
                return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
        }

        *data = bap->oa_value;
        *numelems = bap->oa_value_len;

        return (CRYPTO_SUCCESS);
}

/*
 * Finds entry of specified name. If it is not found dca_find_attribute returns
 * NULL.
 */
crypto_object_attribute_t *
dca_find_attribute(crypto_object_attribute_t *attrp, uint_t atnum,
    uint64_t atype)
{
        while (atnum) {
                if (attrp->oa_type == atype)
                        return (attrp);
                atnum--;
                attrp++;
        }
        return (NULL);
}

/*
 * Return the address of the first data buffer. If the data format is
 * unrecognised return NULL.
 */
caddr_t
dca_bufdaddr(crypto_data_t *data)
{
        switch (data->cd_format) {
        case CRYPTO_DATA_RAW:
                return (data->cd_raw.iov_base + data->cd_offset);
        case CRYPTO_DATA_UIO:
                return (data->cd_uio->uio_iov[0].iov_base + data->cd_offset);
        case CRYPTO_DATA_MBLK:
                return ((char *)data->cd_mp->b_rptr + data->cd_offset);
        default:
                DBG(NULL, DWARN,
                    "dca_bufdaddr: unrecognised crypto data format");
                return (NULL);
        }
}

static caddr_t
dca_bufdaddr_out(crypto_data_t *data)
{
        size_t offset = data->cd_offset + data->cd_length;

        switch (data->cd_format) {
        case CRYPTO_DATA_RAW:
                return (data->cd_raw.iov_base + offset);
        case CRYPTO_DATA_UIO:
                return (data->cd_uio->uio_iov[0].iov_base + offset);
        case CRYPTO_DATA_MBLK:
                return ((char *)data->cd_mp->b_rptr + offset);
        default:
                DBG(NULL, DWARN,
                    "dca_bufdaddr_out: unrecognised crypto data format");
                return (NULL);
        }
}

/*
 * Control entry points.
 */

/* ARGSUSED */
static void
dca_provider_status(crypto_provider_handle_t provider, uint_t *status)
{
        *status = CRYPTO_PROVIDER_READY;
}

/*
 * Cipher (encrypt/decrypt) entry points.
 */

/* ARGSUSED */
static int
dca_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_encrypt_init: started");

        /* check mechanism */
        switch (mechanism->cm_type) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
                    DR_ENCRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
                    DR_ENCRYPT | DR_TRIPLE);
                break;
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
                break;
        default:
                cmn_err(CE_WARN, "dca_encrypt_init: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_encrypt_init: done, err = 0x%x", error);

        if (error == CRYPTO_SUCCESS)
                dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
                    &softc->dca_ctx_list_lock);

        return (error);
}

/* ARGSUSED */
static int
dca_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
    crypto_data_t *ciphertext, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_encrypt: started");

        /* handle inplace ops */
        if (!ciphertext) {
                dca_request_t *reqp = ctx->cc_provider_private;
                reqp->dr_flags |= DR_INPLACE;
                ciphertext = plaintext;
        }

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3des(ctx, plaintext, ciphertext, req,
                    DR_ENCRYPT | DR_TRIPLE);
                break;
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsastart(ctx, plaintext, ciphertext, req,
                    DCA_RSA_ENC);
                break;
        default:
                /* Should never reach here */
                cmn_err(CE_WARN, "dca_encrypt: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
            (error != CRYPTO_BUFFER_TOO_SMALL)) {
                ciphertext->cd_length = 0;
        }

        DBG(softc, DENTRY, "dca_encrypt: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext,
    crypto_data_t *ciphertext, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_encrypt_update: started");

        /* handle inplace ops */
        if (!ciphertext) {
                dca_request_t *reqp = ctx->cc_provider_private;
                reqp->dr_flags |= DR_INPLACE;
                ciphertext = plaintext;
        }

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desupdate(ctx, plaintext, ciphertext, req,
                    DR_ENCRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desupdate(ctx, plaintext, ciphertext, req,
                    DR_ENCRYPT | DR_TRIPLE);
                break;
        default:
                /* Should never reach here */
                cmn_err(CE_WARN, "dca_encrypt_update: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_encrypt_update: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_encrypt_final: started");

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT | DR_TRIPLE);
                break;
        default:
                /* Should never reach here */
                cmn_err(CE_WARN, "dca_encrypt_final: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_encrypt_final: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_encrypt_atomic(crypto_provider_handle_t provider,
    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
    crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc = (dca_t *)provider;

        DBG(softc, DENTRY, "dca_encrypt_atomic: started");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* handle inplace ops */
        if (!ciphertext) {
                ciphertext = plaintext;
        }

        /* check mechanism */
        switch (mechanism->cm_type) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desatomic(provider, session_id, mechanism, key,
                    plaintext, ciphertext, KM_SLEEP, req,
                    DR_ENCRYPT | DR_ATOMIC);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desatomic(provider, session_id, mechanism, key,
                    plaintext, ciphertext, KM_SLEEP, req,
                    DR_ENCRYPT | DR_TRIPLE | DR_ATOMIC);
                break;
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsaatomic(provider, session_id, mechanism, key,
                    plaintext, ciphertext, KM_SLEEP, req, DCA_RSA_ENC);
                break;
        default:
                cmn_err(CE_WARN, "dca_encrypt_atomic: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
                ciphertext->cd_length = 0;
        }

        DBG(softc, DENTRY, "dca_encrypt_atomic: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_decrypt_init: started");

        /* check mechanism */
        switch (mechanism->cm_type) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
                    DR_DECRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
                    DR_DECRYPT | DR_TRIPLE);
                break;
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
                break;
        default:
                cmn_err(CE_WARN, "dca_decrypt_init: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_decrypt_init: done, err = 0x%x", error);

        if (error == CRYPTO_SUCCESS)
                dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
                    &softc->dca_ctx_list_lock);

        return (error);
}

/* ARGSUSED */
static int
dca_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
    crypto_data_t *plaintext, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_decrypt: started");

        /* handle inplace ops */
        if (!plaintext) {
                dca_request_t *reqp = ctx->cc_provider_private;
                reqp->dr_flags |= DR_INPLACE;
                plaintext = ciphertext;
        }

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3des(ctx, ciphertext, plaintext, req,
                    DR_DECRYPT | DR_TRIPLE);
                break;
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsastart(ctx, ciphertext, plaintext, req,
                    DCA_RSA_DEC);
                break;
        default:
                /* Should never reach here */
                cmn_err(CE_WARN, "dca_decrypt: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
            (error != CRYPTO_BUFFER_TOO_SMALL)) {
                if (plaintext)
                        plaintext->cd_length = 0;
        }

        DBG(softc, DENTRY, "dca_decrypt: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
    crypto_data_t *plaintext, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_decrypt_update: started");

        /* handle inplace ops */
        if (!plaintext) {
                dca_request_t *reqp = ctx->cc_provider_private;
                reqp->dr_flags |= DR_INPLACE;
                plaintext = ciphertext;
        }

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desupdate(ctx, ciphertext, plaintext, req,
                    DR_DECRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desupdate(ctx, ciphertext, plaintext, req,
                    DR_DECRYPT | DR_TRIPLE);
                break;
        default:
                /* Should never reach here */
                cmn_err(CE_WARN, "dca_decrypt_update: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_decrypt_update: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *plaintext,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_decrypt_final: started");

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desfinal(ctx, plaintext, DR_DECRYPT);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desfinal(ctx, plaintext, DR_DECRYPT | DR_TRIPLE);
                break;
        default:
                /* Should never reach here */
                cmn_err(CE_WARN, "dca_decrypt_final: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_decrypt_final: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_decrypt_atomic(crypto_provider_handle_t provider,
    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
    crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc = (dca_t *)provider;

        DBG(softc, DENTRY, "dca_decrypt_atomic: started");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* handle inplace ops */
        if (!plaintext) {
                plaintext = ciphertext;
        }

        /* check mechanism */
        switch (mechanism->cm_type) {
        case DES_CBC_MECH_INFO_TYPE:
                error = dca_3desatomic(provider, session_id, mechanism, key,
                    ciphertext, plaintext, KM_SLEEP, req,
                    DR_DECRYPT | DR_ATOMIC);
                break;
        case DES3_CBC_MECH_INFO_TYPE:
                error = dca_3desatomic(provider, session_id, mechanism, key,
                    ciphertext, plaintext, KM_SLEEP, req,
                    DR_DECRYPT | DR_TRIPLE | DR_ATOMIC);
                break;
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsaatomic(provider, session_id, mechanism, key,
                    ciphertext, plaintext, KM_SLEEP, req, DCA_RSA_DEC);
                break;
        default:
                cmn_err(CE_WARN, "dca_decrypt_atomic: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
                plaintext->cd_length = 0;
        }

        DBG(softc, DENTRY, "dca_decrypt_atomic: done, err = 0x%x", error);

        return (error);
}

/*
 * Sign entry points.
 */

/* ARGSUSED */
static int
dca_sign_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_sign_init: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
                break;
        case DSA_MECH_INFO_TYPE:
                error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
                    DCA_DSA_SIGN);
                break;
        default:
                cmn_err(CE_WARN, "dca_sign_init: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_sign_init: done, err = 0x%x", error);

        if (error == CRYPTO_SUCCESS)
                dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
                    &softc->dca_ctx_list_lock);

        return (error);
}

static int
dca_sign(crypto_ctx_t *ctx, crypto_data_t *data,
    crypto_data_t *signature, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_sign: started\n");

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGN);
                break;
        case DSA_MECH_INFO_TYPE:
                error = dca_dsa_sign(ctx, data, signature, req);
                break;
        default:
                cmn_err(CE_WARN, "dca_sign: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_sign: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_sign_update(crypto_ctx_t *ctx, crypto_data_t *data,
    crypto_req_handle_t req)
{
        int error = CRYPTO_MECHANISM_INVALID;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_sign_update: started\n");

        cmn_err(CE_WARN, "dca_sign_update: unexpected mech type "
            "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));

        DBG(softc, DENTRY, "dca_sign_update: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
    crypto_req_handle_t req)
{
        int error = CRYPTO_MECHANISM_INVALID;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_sign_final: started\n");

        cmn_err(CE_WARN, "dca_sign_final: unexpected mech type "
            "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));

        DBG(softc, DENTRY, "dca_sign_final: done, err = 0x%x", error);

        return (error);
}

static int
dca_sign_atomic(crypto_provider_handle_t provider,
    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
    crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc = (dca_t *)provider;

        DBG(softc, DENTRY, "dca_sign_atomic: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsaatomic(provider, session_id, mechanism, key,
                    data, signature, KM_SLEEP, req, DCA_RSA_SIGN);
                break;
        case DSA_MECH_INFO_TYPE:
                error = dca_dsaatomic(provider, session_id, mechanism, key,
                    data, signature, KM_SLEEP, req, DCA_DSA_SIGN);
                break;
        default:
                cmn_err(CE_WARN, "dca_sign_atomic: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_sign_atomic: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_sign_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_sign_recover_init: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
                break;
        default:
                cmn_err(CE_WARN, "dca_sign_recover_init: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_sign_recover_init: done, err = 0x%x", error);

        if (error == CRYPTO_SUCCESS)
                dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
                    &softc->dca_ctx_list_lock);

        return (error);
}

static int
dca_sign_recover(crypto_ctx_t *ctx, crypto_data_t *data,
    crypto_data_t *signature, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_sign_recover: started\n");

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGNR);
                break;
        default:
                cmn_err(CE_WARN, "dca_sign_recover: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_sign_recover: done, err = 0x%x", error);

        return (error);
}

static int
dca_sign_recover_atomic(crypto_provider_handle_t provider,
    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
    crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc = (dca_t *)provider;

        DBG(softc, DENTRY, "dca_sign_recover_atomic: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsaatomic(provider, session_id, mechanism, key,
                    data, signature, KM_SLEEP, req, DCA_RSA_SIGNR);
                break;
        default:
                cmn_err(CE_WARN, "dca_sign_recover_atomic: unexpected mech type"
                    " 0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_sign_recover_atomic: done, err = 0x%x", error);

        return (error);
}

/*
 * Verify entry points.
 */

/* ARGSUSED */
static int
dca_verify_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_verify_init: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
                break;
        case DSA_MECH_INFO_TYPE:
                error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
                    DCA_DSA_VRFY);
                break;
        default:
                cmn_err(CE_WARN, "dca_verify_init: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_verify_init: done, err = 0x%x", error);

        if (error == CRYPTO_SUCCESS)
                dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
                    &softc->dca_ctx_list_lock);

        return (error);
}

static int
dca_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
    crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_verify: started\n");

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFY);
                break;
        case DSA_MECH_INFO_TYPE:
                error = dca_dsa_verify(ctx, data, signature, req);
                break;
        default:
                cmn_err(CE_WARN, "dca_verify: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_verify: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
    crypto_req_handle_t req)
{
        int error = CRYPTO_MECHANISM_INVALID;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_verify_update: started\n");

        cmn_err(CE_WARN, "dca_verify_update: unexpected mech type "
            "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));

        DBG(softc, DENTRY, "dca_verify_update: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
    crypto_req_handle_t req)
{
        int error = CRYPTO_MECHANISM_INVALID;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_verify_final: started\n");

        cmn_err(CE_WARN, "dca_verify_final: unexpected mech type "
            "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));

        DBG(softc, DENTRY, "dca_verify_final: done, err = 0x%x", error);

        return (error);
}

static int
dca_verify_atomic(crypto_provider_handle_t provider,
    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
    crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc = (dca_t *)provider;

        DBG(softc, DENTRY, "dca_verify_atomic: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsaatomic(provider, session_id, mechanism, key,
                    signature, data, KM_SLEEP, req, DCA_RSA_VRFY);
                break;
        case DSA_MECH_INFO_TYPE:
                error = dca_dsaatomic(provider, session_id, mechanism, key,
                    data, signature, KM_SLEEP, req, DCA_DSA_VRFY);
                break;
        default:
                cmn_err(CE_WARN, "dca_verify_atomic: unexpected mech type "
                    "0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY, "dca_verify_atomic: done, err = 0x%x", error);

        return (error);
}

/* ARGSUSED */
static int
dca_verify_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
    crypto_req_handle_t req)
{
        int error = CRYPTO_MECHANISM_INVALID;
        dca_t *softc;

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_verify_recover_init: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
                break;
        default:
                cmn_err(CE_WARN, "dca_verify_recover_init: unexpected mech type"
                    " 0x%llx\n", (unsigned long long)mechanism->cm_type);
        }

        DBG(softc, DENTRY, "dca_verify_recover_init: done, err = 0x%x", error);

        if (error == CRYPTO_SUCCESS)
                dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
                    &softc->dca_ctx_list_lock);

        return (error);
}

static int
dca_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
    crypto_data_t *data, crypto_req_handle_t req)
{
        int error = CRYPTO_MECHANISM_INVALID;
        dca_t *softc;

        if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
                return (CRYPTO_OPERATION_NOT_INITIALIZED);

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_verify_recover: started\n");

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFYR);
                break;
        default:
                cmn_err(CE_WARN, "dca_verify_recover: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
        }

        DBG(softc, DENTRY, "dca_verify_recover: done, err = 0x%x", error);

        return (error);
}

static int
dca_verify_recover_atomic(crypto_provider_handle_t provider,
    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
    crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
    crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
{
        int error = CRYPTO_MECHANISM_INVALID;
        dca_t *softc = (dca_t *)provider;

        DBG(softc, DENTRY, "dca_verify_recover_atomic: started\n");

        if (ctx_template != NULL)
                return (CRYPTO_ARGUMENTS_BAD);

        /* check mechanism */
        switch (mechanism->cm_type) {
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                error = dca_rsaatomic(provider, session_id, mechanism, key,
                    signature, data, KM_SLEEP, req, DCA_RSA_VRFYR);
                break;
        default:
                cmn_err(CE_WARN, "dca_verify_recover_atomic: unexpected mech "
                    "type 0x%llx\n", (unsigned long long)mechanism->cm_type);
                error = CRYPTO_MECHANISM_INVALID;
        }

        DBG(softc, DENTRY,
            "dca_verify_recover_atomic: done, err = 0x%x", error);

        return (error);
}

/*
 * Random number entry points.
 */

/* ARGSUSED */
static int
dca_generate_random(crypto_provider_handle_t provider,
    crypto_session_id_t session_id,
    uchar_t *buf, size_t len, crypto_req_handle_t req)
{
        int error = CRYPTO_FAILED;
        dca_t *softc = (dca_t *)provider;

        DBG(softc, DENTRY, "dca_generate_random: started");

        error = dca_rng(softc, buf, len, req);

        DBG(softc, DENTRY, "dca_generate_random: done, err = 0x%x", error);

        return (error);
}

/*
 * Context management entry points.
 */

int
dca_free_context(crypto_ctx_t *ctx)
{
        int error = CRYPTO_SUCCESS;
        dca_t *softc;

        softc = DCA_SOFTC_FROM_CTX(ctx);
        DBG(softc, DENTRY, "dca_free_context: entered");

        if (ctx->cc_provider_private == NULL)
                return (error);

        dca_rmlist2(ctx->cc_provider_private, &softc->dca_ctx_list_lock);

        error = dca_free_context_low(ctx);

        DBG(softc, DENTRY, "dca_free_context: done, err = 0x%x", error);

        return (error);
}

static int
dca_free_context_low(crypto_ctx_t *ctx)
{
        int error = CRYPTO_SUCCESS;

        /* check mechanism */
        switch (DCA_MECH_FROM_CTX(ctx)) {
        case DES_CBC_MECH_INFO_TYPE:
        case DES3_CBC_MECH_INFO_TYPE:
                dca_3desctxfree(ctx);
                break;
        case RSA_PKCS_MECH_INFO_TYPE:
        case RSA_X_509_MECH_INFO_TYPE:
                dca_rsactxfree(ctx);
                break;
        case DSA_MECH_INFO_TYPE:
                dca_dsactxfree(ctx);
                break;
        default:
                /* Should never reach here */
                cmn_err(CE_WARN, "dca_free_context_low: unexpected mech type "
                    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
                error = CRYPTO_MECHANISM_INVALID;
        }

        return (error);
}


/* Free any unfreed private context. It is called in detach. */
static void
dca_free_context_list(dca_t *dca)
{
        dca_listnode_t  *node;
        crypto_ctx_t    ctx;

        (void) memset(&ctx, 0, sizeof (ctx));
        ctx.cc_provider = dca;

        while ((node = dca_delist2(&dca->dca_ctx_list,
            &dca->dca_ctx_list_lock)) != NULL) {
                ctx.cc_provider_private = node;
                (void) dca_free_context_low(&ctx);
        }
}

static int
ext_info_sym(crypto_provider_handle_t prov,
    crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
{
        return (ext_info_base(prov, ext_info, cfreq, IDENT_SYM));
}

static int
ext_info_asym(crypto_provider_handle_t prov,
    crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
{
        int rv;

        rv = ext_info_base(prov, ext_info, cfreq, IDENT_ASYM);
        /* The asymmetric cipher slot supports random */
        ext_info->ei_flags |= CRYPTO_EXTF_RNG;

        return (rv);
}

/* ARGSUSED */
static int
ext_info_base(crypto_provider_handle_t prov,
    crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id)
{
        dca_t   *dca = (dca_t *)prov;
        int len;

        /* Label */
        (void) sprintf((char *)ext_info->ei_label, "%s/%d %s",
            ddi_driver_name(dca->dca_dip), ddi_get_instance(dca->dca_dip), id);
        len = strlen((char *)ext_info->ei_label);
        (void) memset(ext_info->ei_label + len, ' ',
            CRYPTO_EXT_SIZE_LABEL - len);

        /* Manufacturer ID */
        (void) sprintf((char *)ext_info->ei_manufacturerID, "%s",
            DCA_MANUFACTURER_ID);
        len = strlen((char *)ext_info->ei_manufacturerID);
        (void) memset(ext_info->ei_manufacturerID + len, ' ',
            CRYPTO_EXT_SIZE_MANUF - len);

        /* Model */
        (void) sprintf((char *)ext_info->ei_model, dca->dca_model);

        DBG(dca, DWARN, "kCF MODEL: %s", (char *)ext_info->ei_model);

        len = strlen((char *)ext_info->ei_model);
        (void) memset(ext_info->ei_model + len, ' ',
            CRYPTO_EXT_SIZE_MODEL - len);

        /* Serial Number. Blank for Deimos */
        (void) memset(ext_info->ei_serial_number, ' ', CRYPTO_EXT_SIZE_SERIAL);

        ext_info->ei_flags = CRYPTO_EXTF_WRITE_PROTECTED;

        ext_info->ei_max_session_count = CRYPTO_UNAVAILABLE_INFO;
        ext_info->ei_max_pin_len = CRYPTO_UNAVAILABLE_INFO;
        ext_info->ei_min_pin_len = CRYPTO_UNAVAILABLE_INFO;
        ext_info->ei_total_public_memory = CRYPTO_UNAVAILABLE_INFO;
        ext_info->ei_free_public_memory = CRYPTO_UNAVAILABLE_INFO;
        ext_info->ei_total_private_memory = CRYPTO_UNAVAILABLE_INFO;
        ext_info->ei_free_private_memory = CRYPTO_UNAVAILABLE_INFO;
        ext_info->ei_hardware_version.cv_major = 0;
        ext_info->ei_hardware_version.cv_minor = 0;
        ext_info->ei_firmware_version.cv_major = 0;
        ext_info->ei_firmware_version.cv_minor = 0;

        /* Time. No need to be supplied for token without a clock */
        ext_info->ei_time[0] = '\000';

        return (CRYPTO_SUCCESS);
}

static void
dca_fma_init(dca_t *dca)
{
        ddi_iblock_cookie_t fm_ibc;
        int fm_capabilities = DDI_FM_EREPORT_CAPABLE |
            DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE |
            DDI_FM_ERRCB_CAPABLE;

        /* Read FMA capabilities from dca.conf file (if present) */
        dca->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip,
            DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
            fm_capabilities);

        DBG(dca, DWARN, "dca->fm_capabilities = 0x%x", dca->fm_capabilities);

        /* Only register with IO Fault Services if we have some capability */
        if (dca->fm_capabilities) {
                dca_regsattr.devacc_attr_access = DDI_FLAGERR_ACC;
                dca_dmaattr.dma_attr_flags = DDI_DMA_FLAGERR;

                /* Register capabilities with IO Fault Services */
                ddi_fm_init(dca->dca_dip, &dca->fm_capabilities, &fm_ibc);
                DBG(dca, DWARN, "fm_capable() =  0x%x",
                    ddi_fm_capable(dca->dca_dip));

                /*
                 * Initialize pci ereport capabilities if ereport capable
                 */
                if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
                    DDI_FM_ERRCB_CAP(dca->fm_capabilities))
                        pci_ereport_setup(dca->dca_dip);

                /*
                 * Initialize callback mutex and register error callback if
                 * error callback capable.
                 */
                if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
                        ddi_fm_handler_register(dca->dca_dip, dca_fm_error_cb,
                            (void *)dca);
                }
        } else {
                /*
                 * These fields have to be cleared of FMA if there are no
                 * FMA capabilities at runtime.
                 */
                dca_regsattr.devacc_attr_access = DDI_DEFAULT_ACC;
                dca_dmaattr.dma_attr_flags = 0;
        }
}


static void
dca_fma_fini(dca_t *dca)
{
        /* Only unregister FMA capabilities if we registered some */
        if (dca->fm_capabilities) {

                /*
                 * Release any resources allocated by pci_ereport_setup()
                 */
                if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
                    DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
                        pci_ereport_teardown(dca->dca_dip);
                }

                /*
                 * Free callback mutex and un-register error callback if
                 * error callback capable.
                 */
                if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
                        ddi_fm_handler_unregister(dca->dca_dip);
                }

                /* Unregister from IO Fault Services */
                ddi_fm_fini(dca->dca_dip);
                DBG(dca, DWARN, "fm_capable() = 0x%x",
                    ddi_fm_capable(dca->dca_dip));
        }
}


/*
 * The IO fault service error handling callback function
 */
/*ARGSUSED*/
static int
dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
{
        dca_t           *dca = (dca_t *)impl_data;

        pci_ereport_post(dip, err, NULL);
        if (err->fme_status == DDI_FM_FATAL) {
                dca_failure(dca, DDI_DATAPATH_FAULT,
                    DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
                    "fault PCI in FMA callback.");
        }
        return (err->fme_status);
}


static int
dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
    dca_fma_eclass_t eclass_index)
{
        ddi_fm_error_t  de;
        int             version = 0;

        ddi_fm_acc_err_get(handle, &de, version);
        if (de.fme_status != DDI_FM_OK) {
                dca_failure(dca, DDI_DATAPATH_FAULT,
                    eclass_index, fm_ena_increment(de.fme_ena),
                    CRYPTO_DEVICE_ERROR, "");
                return (DDI_FAILURE);
        }

        return (DDI_SUCCESS);
}

int
dca_check_dma_handle(dca_t *dca, ddi_dma_handle_t handle,
    dca_fma_eclass_t eclass_index)
{
        ddi_fm_error_t  de;
        int             version = 0;

        ddi_fm_dma_err_get(handle, &de, version);
        if (de.fme_status != DDI_FM_OK) {
                dca_failure(dca, DDI_DATAPATH_FAULT,
                    eclass_index, fm_ena_increment(de.fme_ena),
                    CRYPTO_DEVICE_ERROR, "");
                return (DDI_FAILURE);
        }
        return (DDI_SUCCESS);
}

static uint64_t
dca_ena(uint64_t ena)
{
        if (ena == 0)
                ena = fm_ena_generate(0, FM_ENA_FMT1);
        else
                ena = fm_ena_increment(ena);
        return (ena);
}

static char *
dca_fma_eclass_string(char *model, dca_fma_eclass_t index)
{
        if (strstr(model, "500"))
                return (dca_fma_eclass_sca500[index]);
        else
                return (dca_fma_eclass_sca1000[index]);
}