/*      $NetBSD: cryptodev.c,v 1.135 2026/07/05 15:34:13 riastradh Exp $ */
/*      $FreeBSD: src/sys/opencrypto/cryptodev.c,v 1.4.2.4 2003/06/03 00:09:02 sam Exp $        */
/*      $OpenBSD: cryptodev.c,v 1.53 2002/07/10 22:21:30 mickey Exp $   */

/*-
 * Copyright (c) 2008 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Coyote Point Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (c) 2001 Theo de Raadt
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Effort sponsored in part by the Defense Advanced Research Projects
 * Agency (DARPA) and Air Force Research Laboratory, Air Force
 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
 *
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: cryptodev.c,v 1.135 2026/07/05 15:34:13 riastradh Exp $");

#include <sys/param.h>

#include <sys/atomic.h>
#include <sys/compat_stub.h>
#include <sys/conf.h>
#include <sys/device.h>
#include <sys/errno.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/kauth.h>
#include <sys/kmem.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/md5.h>
#include <sys/module.h>
#include <sys/poll.h>
#include <sys/pool.h>
#include <sys/sdt.h>
#include <sys/select.h>
#include <sys/sha1.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

#include "ioconf.h"

#ifdef _KERNEL_OPT
#include "opt_ocf.h"
#include "opt_compat_netbsd.h"
#endif

#include <opencrypto/cryptodev.h>
#include <opencrypto/cryptodev_internal.h>
#include <opencrypto/ocryptodev.h>
#include <opencrypto/xform.h>

#include "ioconf.h"

static kmutex_t cryptodev_mtx;

struct csession {
        TAILQ_ENTRY(csession) next;
        kmutex_t        lock;
        uint64_t        sid;
        uint32_t        ses;
        volatile uint32_t       refcnt;

        uint32_t        cipher;         /* note: shares name space in crd_alg */
        const struct enc_xform *txform;
        uint32_t        mac;            /* note: shares name space in crd_alg */
        const struct auth_hash *thash;
        uint32_t        comp_alg;       /* note: shares name space in crd_alg */
        const struct comp_algo *tcomp;

        void *          key;
        int             keylen;

        void *          mackey;
        int             mackeylen;
};

struct fcrypt {
        TAILQ_HEAD(csessionlist, csession) csessions;
        TAILQ_HEAD(crprethead, cryptop) crp_ret_mq;
        TAILQ_HEAD(krprethead, cryptkop) crp_ret_mkq;
        uint32_t        sesn;
        struct selinfo  sinfo;
        uint32_t        requestid;
        struct timespec atime;
        struct timespec mtime;
        struct timespec btime;
        kmutex_t        lock;
};

/* For our fixed-size allocations */
static struct pool fcrpl;
static struct pool csepl;

/* Declaration of master device (fd-cloning/ctxt-allocating) entrypoints */
static int      cryptoopen(dev_t dev, int flag, int mode, struct lwp *l);
static int      cryptoread(dev_t dev, struct uio *uio, int ioflag);
static int      cryptowrite(dev_t dev, struct uio *uio, int ioflag);
static int      cryptoselect(dev_t dev, int rw, struct lwp *l);

static bool     crypto_detaching = false;
static int      crypto_refcount = 0;    /* Prevent detaching while in use */

static int
crypto_refcount_get(void)
{
        int error;

        mutex_enter(&cryptodev_mtx);
        if (crypto_detaching) {
                error = SET_ERROR(ENXIO);
        } else if (crypto_refcount == INT_MAX) {
                error = SET_ERROR(EBUSY);
        } else {
                crypto_refcount++;
                error = 0;
        }
        mutex_exit(&cryptodev_mtx);

        return error;
}

static void
crypto_refcount_put(void)
{

        mutex_enter(&cryptodev_mtx);
        KASSERT(crypto_refcount > 0);
        crypto_refcount--;
        mutex_exit(&cryptodev_mtx);
}

/* Declaration of cloned-device (per-ctxt) entrypoints */
static int      cryptof_read(struct file *, off_t *, struct uio *,
    kauth_cred_t, int);
static int      cryptof_write(struct file *, off_t *, struct uio *,
    kauth_cred_t, int);
static int      cryptof_ioctl(struct file *, u_long, void *);
static int      cryptof_close(struct file *);
static int      cryptof_poll(struct file *, int);
static int      cryptof_stat(struct file *, struct stat *);

static const struct fileops cryptofops = {
        .fo_name = "cryptof",
        .fo_read = cryptof_read,
        .fo_write = cryptof_write,
        .fo_ioctl = cryptof_ioctl,
        .fo_fcntl = fnullop_fcntl,
        .fo_poll = cryptof_poll,
        .fo_stat = cryptof_stat,
        .fo_close = cryptof_close,
        .fo_kqfilter = fnullop_kqfilter,
        .fo_restart = fnullop_restart,
};

static struct   csession *cse_find(struct fcrypt *, uint32_t);
static int      cse_delete(struct fcrypt *, uint32_t);
static struct   csession *cse_add(struct fcrypt *, struct csession *);
static struct   csession *cse_create(struct fcrypt *, uint64_t, void *,
    uint64_t, void *, uint64_t, uint32_t, uint32_t, uint32_t,
    const struct enc_xform *, const struct auth_hash *,
    const struct comp_algo *);
static void     cse_free(struct csession *);

static int      cryptodev_key(struct crypt_kop *);
static int      cryptodev_mkey(struct fcrypt *, struct crypt_n_kop *, size_t);
static void     cryptodev_msessionfin(struct fcrypt *, size_t, uint32_t *);

static void     cryptodev_cb(struct cryptop *);
static void     cryptodevkey_cb(struct cryptkop *);

static void     cryptodev_mcb(struct cryptop *);
static void     cryptodevkey_mcb(struct cryptkop *);

static int      cryptodev_getmstatus(struct fcrypt *, struct crypt_result *,
    int);
static int      cryptodev_getstatus(struct fcrypt *, struct crypt_result *);

static struct cryptop_data *
cod_ctor(struct cryptop_data *cod, struct csession *cse, size_t iov_len)
{
        memset(cod, 0, sizeof(*cod));
        cod->cse = cse;
        cod->iov_len = iov_len;
        cod->uio.uio_iovcnt = 1;
        cod->uio.uio_resid = 0;
        cod->uio.uio_rw = UIO_WRITE;
        cod->uio.uio_iov = cod->iovec;
        UIO_SETUP_SYSSPACE(&cod->uio);

        /* the iov needs to be big enough to handle the uncompressed data.... */
        cod->uio.uio_iov[0].iov_len = cod->iov_len;
        if (cod->iov_len > 0)
                cod->uio.uio_iov[0].iov_base = kmem_zalloc(iov_len, KM_SLEEP);
        cod->uio.uio_resid = cod->uio.uio_iov[0].iov_len;
        DPRINTF("lid[%u]: uio.iov_base %p malloced %zu bytes\n",
                CRYPTO_SESID2LID(cse->sid),
                cod->uio.uio_iov[0].iov_base, cod->iov_len);
        return cod;
}

static void
cod_dtor(struct cryptop_data *cod)
{
        DPRINTF("lid[%u]: uio.iov_base %p freeing %zu bytes\n",
                CRYPTO_SESID2LID(cod->cse->sid),
                cod->uio.uio_iov[0].iov_base, cod->iov_len);
        if (cod->iov_len && cod->uio.uio_iov[0].iov_base) {
                kmem_free(cod->uio.uio_iov[0].iov_base, cod->iov_len);
                cod->uio.uio_iov[0].iov_base = NULL;
        }
}

static void
fcrypt_settime(struct fcrypt *fcr)
{
        mutex_enter(&fcr->lock);
        fcr->mtime = fcr->atime;
        mutex_exit(&fcr->lock);
}

static struct fcrypt *
fcrypt_ctor(void)
{
        struct fcrypt *fcr;

        fcr = pool_get(&fcrpl, PR_WAITOK);
        getnanotime(&fcr->btime);
        fcr->atime = fcr->mtime = fcr->btime;
        TAILQ_INIT(&fcr->csessions);
        TAILQ_INIT(&fcr->crp_ret_mq);
        TAILQ_INIT(&fcr->crp_ret_mkq);
        selinit(&fcr->sinfo);
        /*
         * Don't ever return session 0, to allow detection of
         * failed creation attempts with multi-create ioctl.
         */
        fcr->sesn = 1;
        fcr->requestid = 1;
        mutex_init(&fcr->lock, MUTEX_DEFAULT, IPL_NONE);

        return fcr;
}

static void
fcrypt_dtor(struct fcrypt *fcr)
{
        struct csession *cse;

        while ((cse = TAILQ_FIRST(&fcr->csessions))) {
                TAILQ_REMOVE(&fcr->csessions, cse, next);
                KASSERT(atomic_load_relaxed(&cse->refcnt) == 1);
                cse_free(cse);
        }
        seldestroy(&fcr->sinfo);
        mutex_destroy(&fcr->lock);

        pool_put(&fcrpl, fcr);
}

/*
 * sysctl-able control variables for /dev/crypto now defined in crypto.c:
 * crypto_usercrypto, crypto_userasymcrypto, crypto_devallowsoft.
 */

/* ARGSUSED */
int
cryptof_read(file_t *fp, off_t *poff,
    struct uio *uio, kauth_cred_t cred, int flags)
{
        return SET_ERROR(EIO);
}

/* ARGSUSED */
int
cryptof_write(file_t *fp, off_t *poff,
    struct uio *uio, kauth_cred_t cred, int flags)
{
        return SET_ERROR(EIO);
}

/* ARGSUSED */
int
cryptof_ioctl(struct file *fp, u_long cmd, void *data)
{
        struct fcrypt *fcr = fp->f_fcrypt;
        struct csession *cse;
        struct session_op *sop;
        struct session_n_op *snop;
        struct crypt_op *cop;
        struct crypt_mop *mop;
        struct crypt_mkop *mkop;
        struct crypt_n_op *cnop;
        struct crypt_n_kop *knop;
        struct crypt_sgop *sgop;
        struct crypt_sfop *sfop;
        struct cryptret *crypt_ret;
        struct crypt_result *crypt_res;
        uint32_t ses;
        uint32_t *sesid;
        int error = 0;
        size_t count, len;

        /* backwards compatibility */
        file_t *criofp;
        struct fcrypt *criofcr;
        int criofd;

        mutex_enter(&fcr->lock);
        getnanotime(&fcr->atime);
        mutex_exit(&fcr->lock);

        switch (cmd) {
        case CRIOGET:   /* XXX deprecated, remove after 5.0 */
                error = crypto_refcount_get();
                if (error)
                        return error;
                if ((error = fd_allocfile(&criofp, &criofd)) != 0) {
                        crypto_refcount_put();
                        return error;
                }
                criofcr = fcrypt_ctor();
                error = fd_clone(criofp, criofd, (FREAD|FWRITE),
                    &cryptofops, criofcr);
                KASSERTMSG(error == EMOVEFD, "error=%d", error);
                *(uint32_t *)data = criofd;
                return 0;
        case CIOCGSESSION:
                sop = (struct session_op *)data;
                error = cryptodev_session(fcr, sop);
                DPRINTF("create(%#x) error = %d\n", sop->ses, error);
                break;
        case CIOCNGSESSION:
                sgop = (struct crypt_sgop *)data;
                if (sgop->count <= 0 || sgop->count >
                    MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*snop))) {
                        error = SET_ERROR(EINVAL);
                        break;
                }
                len = sgop->count * sizeof(*snop);
                snop = kmem_alloc(len, KM_SLEEP);
                error = copyin(sgop->sessions, snop, len);
                if (!error) {
                        fcrypt_settime(fcr);
                        error = cryptodev_msession(fcr, snop, sgop->count);
                        if (!error) {
                                error = copyout(snop, sgop->sessions, len);
                        }
                }
                kmem_free(snop, len);
                break;
        case CIOCFSESSION:
                fcrypt_settime(fcr);
                ses = *(uint32_t *)data;
                mutex_enter(&fcr->lock);
                error = cse_delete(fcr, ses);
                mutex_exit(&fcr->lock);
                DPRINTF("destroy(%#x) error = %d\n", ses, error);
                break;
        case CIOCNFSESSION:
                fcrypt_settime(fcr);
                sfop = (struct crypt_sfop *)data;
                if (sfop->count <= 0 || sfop->count >
                    MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*sesid))) {
                        error = SET_ERROR(EINVAL);
                        break;
                }
                len = sfop->count * sizeof(*sesid);
                sesid = kmem_alloc(len, KM_SLEEP);
                error = copyin(sfop->sesid, sesid, len);
                if (!error) {
                        cryptodev_msessionfin(fcr, sfop->count, sesid);
                }
                kmem_free(sesid, len);
                break;
        case CIOCCRYPT:
                fcrypt_settime(fcr);
                cop = (struct crypt_op *)data;
                cse = cse_find(fcr, cop->ses);
                if (cse == NULL) {
                        DPRINTF("cse_find failed %#x\n", cop->ses);
                        return SET_ERROR(EINVAL);
                }
                error = cryptodev_op(cse, cop, curlwp);
                cse_free(cse);
                DPRINTF("cryptodev_op error = %d\n", error);
                break;
        case CIOCNCRYPTM:
                fcrypt_settime(fcr);
                mop = (struct crypt_mop *)data;
                if (mop->count <= 0 || mop->count >
                    MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*cnop))) {
                        error = SET_ERROR(EINVAL);
                        break;
                }
                len = mop->count * sizeof(*cnop);
                cnop = kmem_alloc(len, KM_SLEEP);
                error = copyin(mop->reqs, cnop, len);
                if (!error) {
                        error = cryptodev_mop(fcr, cnop, mop->count, curlwp);
                        if (!error) {
                                error = copyout(cnop, mop->reqs, len);
                        }
                }
                kmem_free(cnop, len);
                break;
        case CIOCKEY:
                error = cryptodev_key((struct crypt_kop *)data);
                DPRINTF("cryptodev_key error = %d\n", error);
                break;
        case CIOCNFKEYM:
                fcrypt_settime(fcr);
                mkop = (struct crypt_mkop *)data;
                if (mkop->count <= 0 || mkop->count >
                    MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*knop))) {
                        error = SET_ERROR(EINVAL);
                        break;
                }
                len = mkop->count * sizeof(*knop);
                knop = kmem_alloc(len, KM_SLEEP);
                error = copyin(mkop->reqs, knop, len);
                if (!error) {
                        error = cryptodev_mkey(fcr, knop, mkop->count);
                        if (!error)
                                error = copyout(knop, mkop->reqs, len);
                }
                kmem_free(knop, len);
                break;
        case CIOCASYMFEAT:
                error = crypto_getfeat((int *)data);
                break;
        case CIOCNCRYPTRETM:
                fcrypt_settime(fcr);
                crypt_ret = (struct cryptret *)data;
                if (crypt_ret->count <= 0 || crypt_ret->count >
                    MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*crypt_res))) {
                        error = SET_ERROR(EINVAL);
                        break;
                }
                len = crypt_ret->count * sizeof(*crypt_res);
                crypt_res = kmem_alloc(len, KM_SLEEP);
                error = copyin(crypt_ret->results, crypt_res, len);
                if (error)
                        goto reterr;
                count = crypt_ret->count;
                crypt_ret->count = cryptodev_getmstatus(fcr, crypt_res,
                    crypt_ret->count);
                /* sanity check count */
                if (crypt_ret->count > count) {
                        printf("%s.%d: error returned count %zd > original "
                            " count %zd\n",
                            __FILE__, __LINE__, crypt_ret->count, count);
                        crypt_ret->count = count;

                }
                error = copyout(crypt_res, crypt_ret->results, len);
reterr:
                kmem_free(crypt_res, len);
                break;
        case CIOCNCRYPTRET:
                error = cryptodev_getstatus(fcr, (struct crypt_result *)data);
                break;
        default:
                /* Check for backward compatible commands */

                MODULE_HOOK_CALL(ocryptof_50_hook, (fp, cmd, data),
                    enosys(), error);
                if (error == ENOSYS)
                        error = SET_ERROR(EINVAL);
                return error;
        }
        return error;
}

int
cryptodev_op(struct csession *cse, struct crypt_op *cop, struct lwp *l)
{
        struct cryptop *crp = NULL;
        struct cryptop_data *cod = NULL;
        struct cryptodesc *crde = NULL, *crda = NULL, *crdc = NULL;
        int error;
        size_t iov_len = cop->len;
        int flags=0;
        size_t dst_len; /* copyout size */

        if (cop->len > 256*1024-4)
                return E2BIG;

        if (cse->txform) {
                if (cop->len < cse->txform->blocksize
                    + (cop->iv ? 0 : cse->txform->ivsize) ||
                    (cop->len - (cop->iv ? 0 : cse->txform->ivsize))
                    % cse->txform->blocksize != 0)
                        return SET_ERROR(EINVAL);
        }

        if (cse->tcomp == NULL && cse->txform == NULL && cse->thash == NULL)
                return SET_ERROR(EINVAL);

        DPRINTF("[%u]: iov_len %zu\n",
                CRYPTO_SESID2LID(cse->sid), iov_len);
        if ((cse->tcomp) && cop->dst_len) {
                if (iov_len < cop->dst_len) {
                        /* Need larger iov to deal with decompress */
                        iov_len = cop->dst_len;
                }
                DPRINTF("iov_len -> %zu for decompress\n", iov_len);
        }


        crp = crypto_getreq((cse->tcomp != NULL)
            + (cse->txform != NULL) + (cse->thash != NULL));
        if (crp == NULL) {
                error = SET_ERROR(ENOMEM);
                goto bail;
        }
        cod = cod_ctor(&crp->cod, cse, iov_len);
        DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(cse->sid), crp);

        /* crds are always ordered tcomp, thash, then txform */
        /* with optional missing links */

        /* XXX: If we're going to compress then hash or encrypt, we need
         * to be able to pass on the new size of the data.
         */

        if (cse->tcomp) {
                crdc = crp->crp_desc;
        }

        if (cse->thash) {
                crda = crdc ? crdc->crd_next : crp->crp_desc;
                if (cse->txform && crda)
                        crde = crda->crd_next;
        } else {
                if (cse->txform) {
                        crde = crdc ? crdc->crd_next : crp->crp_desc;
                } else if (!cse->tcomp) {
                        error = SET_ERROR(EINVAL);
                        goto bail;
                }
        }

        DPRINTF("ocf[%u]: iov_len %zu, cop->len %u\n",
                        CRYPTO_SESID2LID(cse->sid),
                        cod->uio.uio_iov[0].iov_len,
                        cop->len);

        if ((error = copyin(cop->src, cod->uio.uio_iov[0].iov_base, cop->len)))
        {
                printf("copyin failed %s %d \n", (char *)cop->src, error);
                goto bail;
        }

        if (crdc) {
                switch (cop->op) {
                case COP_COMP:
                        crdc->crd_flags |= CRD_F_COMP;
                        break;
                case COP_DECOMP:
                        crdc->crd_flags &= ~CRD_F_COMP;
                        break;
                default:
                        break;
                }
                /* more data to follow? */
                if (cop->flags & COP_F_MORE) {
                        flags |= CRYPTO_F_MORE;
                }
                crdc->crd_len = cop->len;
                crdc->crd_inject = 0;

                crdc->crd_alg = cse->comp_alg;
                crdc->crd_key = NULL;
                crdc->crd_klen = 0;
                DPRINTF("lid[%u]: crdc setup for comp_alg %d.\n",
                        CRYPTO_SESID2LID(cse->sid), crdc->crd_alg);
        }

        if (crda) {
                crda->crd_skip = 0;
                crda->crd_len = cop->len;
                crda->crd_inject = 0;   /* ??? */

                crda->crd_alg = cse->mac;
                crda->crd_key = cse->mackey;
                crda->crd_klen = cse->mackeylen * 8;
                DPRINTF("crda setup for mac %d.\n", crda->crd_alg);
        }

        if (crde) {
                switch (cop->op) {
                case COP_ENCRYPT:
                        crde->crd_flags |= CRD_F_ENCRYPT;
                        break;
                case COP_DECRYPT:
                        crde->crd_flags &= ~CRD_F_ENCRYPT;
                        break;
                default:
                        break;
                }
                crde->crd_len = cop->len;
                crde->crd_inject = 0;

                if (cse->cipher == CRYPTO_AES_GCM_16 && crda)
                        crda->crd_len = 0;
                else if (cse->cipher == CRYPTO_AES_GMAC)
                        crde->crd_len = 0;

                crde->crd_alg = cse->cipher;
                crde->crd_key = cse->key;
                crde->crd_klen = cse->keylen * 8;
                DPRINTF("crde setup for cipher %d.\n", crde->crd_alg);
        }


        crp->crp_ilen = cop->len;
        crp->crp_flags = CRYPTO_F_IOV | (cop->flags & COP_F_BATCH) | flags;
        crp->crp_buf = (void *)&cod->uio;
        crp->crp_callback = cryptodev_cb;
        crp->crp_sid = cse->sid;
        crp->crp_opaque = cod;

        if (cop->iv) {
                if (crde == NULL) {
                        error = SET_ERROR(EINVAL);
                        goto bail;
                }
                if (cse->txform->ivsize == 0) {
                        error = SET_ERROR(EINVAL);
                        goto bail;
                }
                if ((error = copyin(cop->iv, crp->tmp_iv,
                    cse->txform->ivsize)))
                        goto bail;
                (void)memcpy(crde->crd_iv, crp->tmp_iv, cse->txform->ivsize);
                crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
                crde->crd_skip = 0;
        } else if (crde) {
                if (cse->txform->ivsize == 0) {
                        crde->crd_skip = 0;
                } else {
                        if (!(crde->crd_flags & CRD_F_ENCRYPT))
                                crde->crd_flags |= CRD_F_IV_PRESENT;
                        crde->crd_skip = cse->txform->ivsize;
                        crde->crd_len -= cse->txform->ivsize;
                }
        }

        if (cop->mac) {
                if (crda == NULL) {
                        error = SET_ERROR(EINVAL);
                        goto bail;
                }
                crp->crp_mac = crp->tmp_mac;
        }

        cv_init(&crp->crp_cv, "crydev");
        crypto_dispatch(crp);
        mutex_enter(&cse->lock);
        while (!(crp->crp_devflags & CRYPTODEV_F_RET)) {
                DPRINTF("cse->sid[%u]: sleeping on cv %p for crp %p\n",
                        (uint32_t)cse->sid, &crp->crp_cv, crp);
                cv_wait(&crp->crp_cv, &cse->lock);      /* XXX cv_wait_sig? */
        }
        mutex_exit(&cse->lock);
        cv_destroy(&crp->crp_cv);

        if (crp->crp_etype != 0) {
                DPRINTF("crp_etype %d\n", crp->crp_etype);
                error = crp->crp_etype;
                goto bail;
        }

        dst_len = crp->crp_ilen;
        /* let the user know how much data was returned */
        if (crp->crp_olen) {
                if (crp->crp_olen > (cop->dst_len ? cop->dst_len : cop->len)) {
                        error = SET_ERROR(ENOSPC);
                        goto bail;
                }
                dst_len = cop->dst_len = crp->crp_olen;
        }

        if (cop->dst) {
                DPRINTF("copyout %zu bytes to %p\n", dst_len, cop->dst);
        }
        if (cop->dst &&
            (error = copyout(cod->uio.uio_iov[0].iov_base, cop->dst, dst_len)))
        {
                DPRINTF("copyout error %d\n", error);
                goto bail;
        }

        if (cop->mac &&
            (error = copyout(crp->crp_mac, cop->mac, cse->thash->authsize))) {
                DPRINTF("mac copyout error %d\n", error);
                goto bail;
        }


bail:
        if (cod)
                cod_dtor(cod);
        if (crp)
                crypto_freereq(crp);
        return error;
}

static void
cryptodev_cb(struct cryptop *crp)
{
        struct cryptop_data *cod = crp->crp_opaque;
        DPRINTF("cod = %p\n", cod);

        mutex_enter(&cod->cse->lock);
        crp->crp_devflags |= CRYPTODEV_F_RET;
        cv_signal(&crp->crp_cv);
        mutex_exit(&cod->cse->lock);
}

static void
cryptodev_mcb(struct cryptop *crp)
{
        struct cryptop_data *cod = crp->crp_opaque;

        mutex_enter(&cod->cse->lock);
        TAILQ_INSERT_TAIL(&crp->fcrp->crp_ret_mq, crp, crp_next);
        selnotify(&crp->fcrp->sinfo, 0, 0);
        mutex_exit(&cod->cse->lock);
}

static void
cryptodevkey_cb(struct cryptkop *krp)
{

        mutex_enter(&krp->krp_lock);
        krp->krp_devflags |= CRYPTODEV_F_RET;
        cv_signal(&krp->krp_cv);
        mutex_exit(&krp->krp_lock);
}

static void
cryptodevkey_mcb(struct cryptkop *krp)
{

        mutex_enter(&krp->krp_lock);
        cv_signal(&krp->krp_cv);
        TAILQ_INSERT_TAIL(&krp->fcrp->crp_ret_mkq, krp, krp_next);
        selnotify(&krp->fcrp->sinfo, 0, 0);
        mutex_exit(&krp->krp_lock);
}

static int
cryptodev_key(struct crypt_kop *kop)
{
        struct cryptkop *krp = NULL;
        int error = EINVAL;
        size_t in, out, size, i;

        if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM)
                return SET_ERROR(EFBIG);

        in = kop->crk_iparams;
        out = kop->crk_oparams;
        switch (kop->crk_op) {
        case CRK_MOD_EXP:
                if (in == 3 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_MOD_EXP_CRT:
                if (in == 6 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_DSA_SIGN:
                if (in == 5 && out == 2)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_DSA_VERIFY:
                if (in == 7 && out == 0)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_DH_COMPUTE_KEY:
                if (in == 3 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_MOD_ADD:
                if (in == 3 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_MOD_ADDINV:
                if (in == 2 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_MOD_SUB:
                if (in == 3 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_MOD_MULT:
                if (in == 3 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_MOD_MULTINV:
                if (in == 2 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        case CRK_MOD:
                if (in == 2 && out == 1)
                        break;
                return SET_ERROR(EINVAL);
        default:
                return SET_ERROR(EINVAL);
        }

        krp = crypto_kgetreq(1, PR_WAITOK);
        if (krp == NULL) {
                /* limited by opencrypto.crypto_ret_kq.maxlen */
                return SET_ERROR(ENOMEM);
        }
        (void)memset(krp, 0, sizeof *krp);
        cv_init(&krp->krp_cv, "crykdev");
        mutex_init(&krp->krp_lock, MUTEX_DEFAULT, IPL_NONE);
        krp->krp_op = kop->crk_op;
        krp->krp_status = kop->crk_status;
        krp->krp_iparams = kop->crk_iparams;
        krp->krp_oparams = kop->crk_oparams;
        krp->krp_status = 0;
        krp->krp_callback = cryptodevkey_cb;

        for (i = 0; i < CRK_MAXPARAM; i++)
                krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits;
        for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
                size = (krp->krp_param[i].crp_nbits + 7) / 8;
                if (size == 0)
                        continue;
                krp->krp_param[i].crp_p = kmem_alloc(size, KM_SLEEP);
                if (i >= krp->krp_iparams)
                        continue;
                error = copyin(kop->crk_param[i].crp_p,
                    krp->krp_param[i].crp_p, size);
                if (error)
                        goto fail;
        }

        crypto_kdispatch(krp);

        mutex_enter(&krp->krp_lock);
        while (!(krp->krp_devflags & CRYPTODEV_F_RET)) {
                cv_wait(&krp->krp_cv, &krp->krp_lock);  /* XXX cv_wait_sig? */
        }
        mutex_exit(&krp->krp_lock);

        if (krp->krp_status != 0) {
                DPRINTF("krp->krp_status 0x%08x\n", krp->krp_status);
                error = krp->krp_status;
                goto fail;
        }

        for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams;
            i++) {
                size = (krp->krp_param[i].crp_nbits + 7) / 8;
                if (size == 0)
                        continue;
                error = copyout(krp->krp_param[i].crp_p,
                    kop->crk_param[i].crp_p, size);
                if (error) {
                        DPRINTF("copyout oparam %zu failed, "
                            "error=%d\n", i-krp->krp_iparams, error);
                        goto fail;
                }
        }

fail:
        kop->crk_status = krp->krp_status;
        for (i = 0; i < CRK_MAXPARAM; i++) {
                struct crparam *kp = &(krp->krp_param[i]);
                if (krp->krp_param[i].crp_p) {
                        size = (kp->crp_nbits + 7)  / 8;
                        KASSERT(size > 0);
                        (void)memset(kp->crp_p, 0, size);
                        kmem_free(kp->crp_p, size);
                }
        }
        cv_destroy(&krp->krp_cv);
        crypto_kfreereq(krp);
        DPRINTF("error=0x%08x\n", error);
        return error;
}


/* ARGSUSED */
static int
cryptof_close(struct file *fp)
{
        struct fcrypt *fcr = fp->f_fcrypt;

        fp->f_fcrypt = NULL;
        fcrypt_dtor(fcr);
        crypto_refcount_put();
        return 0;
}

/* needed for compatibility module */
struct  csession *
cryptodev_cse_find(struct fcrypt *fcr, uint32_t ses)
{
        return cse_find(fcr, ses);
}

/* needed for compatibility module */
void
cryptodev_cse_free(struct csession *cse)
{
        cse_free(cse);
}

static struct csession *
cse_find(struct fcrypt *fcr, uint32_t ses)
{
        struct csession *cse, *cnext;

        mutex_enter(&fcr->lock);
        TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext)
                if (cse->ses == ses) {
                        if (atomic_inc_uint_nv(&cse->refcnt) == UINT_MAX) {
                                atomic_dec_uint(&cse->refcnt);
                                cse = NULL;
                        }
                        mutex_exit(&fcr->lock);
                        return cse;
                }
        mutex_exit(&fcr->lock);

        return NULL;
}

static int
cse_delete(struct fcrypt *fcr, uint32_t ses)
{
        struct csession *cse, *cnext;

        TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext) {
                if (cse->ses == ses) {
                        if (atomic_load_relaxed(&cse->refcnt) != 1)
                                return SET_ERROR(EBUSY);
                        TAILQ_REMOVE(&fcr->csessions, cse, next);
                        cse_free(cse);
                        return 0;
                }
        }
        return SET_ERROR(EINVAL);
}

static struct csession *
cse_add(struct fcrypt *fcr, struct csession *cse)
{
        mutex_enter(&fcr->lock);
        /* don't let session ID wrap! */
        if (fcr->sesn + 1 == 0) {
                mutex_exit(&fcr->lock);
                return NULL;
        }
        TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
        cse->ses = fcr->sesn++;
        mutex_exit(&fcr->lock);
        return cse;
}

static struct csession *
cse_create(struct fcrypt *fcr, uint64_t sid, void *key, uint64_t keylen,
    void *mackey, uint64_t mackeylen, uint32_t cipher, uint32_t mac,
    uint32_t comp_alg, const struct enc_xform *txform,
    const struct auth_hash *thash, const struct comp_algo *tcomp)
{
        struct csession *cse;

        cse = pool_get(&csepl, PR_NOWAIT);
        if (cse == NULL)
                return NULL;
        cse->key = key;
        mutex_init(&cse->lock, MUTEX_DEFAULT, IPL_NONE);
        cse->refcnt = 1;
        cse->keylen = keylen/8;
        cse->mackey = mackey;
        cse->mackeylen = mackeylen/8;
        cse->sid = sid;
        cse->cipher = cipher;
        cse->mac = mac;
        cse->comp_alg = comp_alg;
        cse->txform = txform;
        cse->thash = thash;
        cse->tcomp = tcomp;
        if (cse_add(fcr, cse)) {
                DPRINTF("created %p\n", cse);
                return cse;
        }

        mutex_destroy(&cse->lock);
        pool_put(&csepl, cse);
        return NULL;
}

static void
cse_free(struct csession *cse)
{

        DPRINTF("destroying %p %d\n", cse, atomic_load_relaxed(&cse->refcnt));
        membar_release();
        if (atomic_dec_uint_nv(&cse->refcnt) > 0)
                return;
        membar_acquire();
        mutex_destroy(&cse->lock);
        crypto_freesession(cse->sid);
        if (cse->key)
                free(cse->key, M_XDATA);
        if (cse->mackey)
                free(cse->mackey, M_XDATA);
        pool_put(&csepl, cse);
}

static int
cryptoopen(dev_t dev, int flag, int mode,
    struct lwp *l)
{
        file_t *fp;
        struct fcrypt *fcr;
        int fd, error;

        if (crypto_usercrypto == 0)
                return SET_ERROR(ENXIO);

        if ((error = crypto_refcount_get()) != 0)
                return error;

        if ((error = fd_allocfile(&fp, &fd)) != 0) {
                crypto_refcount_put();
                return error;
        }

        fcr = fcrypt_ctor();
        error = fd_clone(fp, fd, flag, &cryptofops, fcr);
        KASSERTMSG(error == EMOVEFD, "error=%d", error);
        return error;
}

static int
cryptoread(dev_t dev, struct uio *uio, int ioflag)
{
        return SET_ERROR(EIO);
}

static int
cryptowrite(dev_t dev, struct uio *uio, int ioflag)
{
        return SET_ERROR(EIO);
}

int
cryptoselect(dev_t dev, int rw, struct lwp *l)
{
        return 0;
}

/*static*/
struct cdevsw crypto_cdevsw = {
        .d_open = cryptoopen,
        .d_close = noclose,
        .d_read = cryptoread,
        .d_write = cryptowrite,
        .d_ioctl = noioctl,
        .d_stop = nostop,
        .d_tty = notty,
        .d_poll = cryptoselect /*nopoll*/,
        .d_mmap = nommap,
        .d_kqfilter = nokqfilter,
        .d_discard = nodiscard,
        .d_flag = D_OTHER
};

int
cryptodev_mop(struct fcrypt *fcr,
              struct crypt_n_op * cnop,
              size_t count, struct lwp *l)
{
        struct cryptop *crp = NULL;
        struct cryptop_data *cod = NULL;
        struct cryptodesc *crde = NULL, *crda = NULL, *crdc = NULL;
        int error = 0;
        struct csession *cse;
        int flags=0;
        size_t req, iov_len;


        for (req = 0; req < count; req++) {
                cse = cse_find(fcr, cnop[req].ses);
                if (cse == NULL) {
                        DPRINTF("cse_find failed %#x\n", cnop[req].ses);
                        cnop[req].status = SET_ERROR(EINVAL);
                        continue;
                }

                if (cnop[req].len > 256*1024-4) {
                        DPRINTF("length failed\n");
                        cnop[req].status = SET_ERROR(EINVAL);
                        continue;
                }
                if (cse->txform) {
                        if (cnop[req].len < cse->txform->blocksize -
                            (cnop[req].iv ? 0 : cse->txform->ivsize) ||
                            (cnop[req].len -
                             (cnop[req].iv ? 0 : cse->txform->ivsize))
                            % cse->txform->blocksize) {
                                cnop[req].status = SET_ERROR(EINVAL);
                                continue;
                        }
                }

                if (cse->txform == NULL &&
                    cse->thash == NULL &&
                    cse->tcomp == NULL) {
                        cnop[req].status = SET_ERROR(EINVAL);
                        goto bail;
                }

                /* sanitize */
                if (cnop[req].len <= 0) {
                        cnop[req].status = SET_ERROR(ENOMEM);
                        goto bail;
                }

                crp = crypto_getreq((cse->txform != NULL) +
                                    (cse->thash != NULL) +
                                    (cse->tcomp != NULL));
                if (crp == NULL) {
                        cnop[req].status = SET_ERROR(ENOMEM);
                        goto bail;
                }

                iov_len = cnop[req].len;
                /* got a compression/decompression max size? */
                if ((cse->tcomp) && cnop[req].dst_len) {
                        if (iov_len < cnop[req].dst_len) {
                                /* Need larger iov to deal with decompress */
                                iov_len = cnop[req].dst_len;
                        }
                        DPRINTF("iov_len -> %zu for decompress\n", iov_len);
                }

                cod = cod_ctor(&crp->cod, cse, iov_len);

                if (cse->tcomp) {
                        crdc = crp->crp_desc;
                }

                if (cse->thash) {
                        crda = crdc ? crdc->crd_next : crp->crp_desc;
                        if (cse->txform && crda)
                                crde = crda->crd_next;
                } else {
                        if (cse->txform) {
                                crde = crdc ? crdc->crd_next : crp->crp_desc;
                        } else if (!cse->tcomp) {
                                error = SET_ERROR(EINVAL);
                                goto bail;
                        }
                }

                if ((copyin(cnop[req].src,
                    cod->uio.uio_iov[0].iov_base, cnop[req].len))) {
                        cnop[req].status = SET_ERROR(EINVAL);
                        goto bail;
                }

                if (crdc) {
                        switch (cnop[req].op) {
                        case COP_COMP:
                                crdc->crd_flags |= CRD_F_COMP;
                                break;
                        case COP_DECOMP:
                                crdc->crd_flags &= ~CRD_F_COMP;
                                break;
                        default:
                                break;
                        }
                        /* more data to follow? */
                        if (cnop[req].flags & COP_F_MORE) {
                                flags |= CRYPTO_F_MORE;
                        }
                        crdc->crd_len = cnop[req].len;
                        crdc->crd_inject = 0;

                        crdc->crd_alg = cse->comp_alg;
                        crdc->crd_key = NULL;
                        crdc->crd_klen = 0;
                        DPRINTF("cse->sid[%d]: crdc setup for comp_alg %d"
                                 " len %d.\n",
                                (uint32_t)cse->sid, crdc->crd_alg,
                                crdc->crd_len);
                }

                if (crda) {
                        crda->crd_skip = 0;
                        crda->crd_len = cnop[req].len;
                        crda->crd_inject = 0;   /* ??? */

                        crda->crd_alg = cse->mac;
                        crda->crd_key = cse->mackey;
                        crda->crd_klen = cse->mackeylen * 8;
                }

                if (crde) {
                        if (cnop[req].op == COP_ENCRYPT)
                                crde->crd_flags |= CRD_F_ENCRYPT;
                        else
                                crde->crd_flags &= ~CRD_F_ENCRYPT;
                        crde->crd_len = cnop[req].len;
                        crde->crd_inject = 0;

                        crde->crd_alg = cse->cipher;
#ifdef notyet           /* XXX must notify h/w driver new key, drain */
                        if(cnop[req].key && cnop[req].keylen) {
                                crde->crd_key = malloc(cnop[req].keylen,
                                                    M_XDATA, M_WAITOK);
                                if((error = copyin(cnop[req].key,
                                    crde->crd_key, cnop[req].keylen))) {
                                        cnop[req].status = SET_ERROR(EINVAL);
                                        goto bail;
                                }
                                crde->crd_klen =  cnop[req].keylen * 8;
                        } else { ... }
#endif
                        crde->crd_key = cse->key;
                        crde->crd_klen = cse->keylen * 8;
                }

                crp->crp_ilen = cnop[req].len;
                crp->crp_flags = CRYPTO_F_IOV |
                    (cnop[req].flags & COP_F_BATCH) | flags;
                crp->crp_buf = (void *)&cod->uio;
                crp->crp_callback = cryptodev_mcb;
                crp->crp_sid = cse->sid;
                crp->crp_opaque = cod;
                crp->fcrp = fcr;
                crp->dst = cnop[req].dst;
                crp->len = cnop[req].len; /* input len, iov may be larger */
                crp->mac = cnop[req].mac;
                DPRINTF("iov_base %p dst %p len %d mac %p\n",
                            cod->uio.uio_iov[0].iov_base, crp->dst, crp->len,
                            crp->mac);

                if (cnop[req].iv) {
                        if (crde == NULL) {
                                cnop[req].status = SET_ERROR(EINVAL);
                                goto bail;
                        }
                        if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
                                cnop[req].status = SET_ERROR(EINVAL);
                                goto bail;
                        }
                        if ((error = copyin(cnop[req].iv, crp->tmp_iv,
                            cse->txform->ivsize))) {
                                cnop[req].status = SET_ERROR(EINVAL);
                                goto bail;
                        }
                        (void)memcpy(crde->crd_iv, crp->tmp_iv,
                            cse->txform->ivsize);
                        crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
                        crde->crd_skip = 0;
                } else if (crde) {
                        if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
                                crde->crd_skip = 0;
                        } else {
                                if (!(crde->crd_flags & CRD_F_ENCRYPT))
                                        crde->crd_flags |= CRD_F_IV_PRESENT;
                                crde->crd_skip = cse->txform->ivsize;
                                crde->crd_len -= cse->txform->ivsize;
                        }
                }

                if (cnop[req].mac) {
                        if (crda == NULL) {
                                cnop[req].status = SET_ERROR(EINVAL);
                                goto bail;
                        }
                        crp->crp_mac = crp->tmp_mac;
                }
                cnop[req].reqid = atomic_inc_32_nv(&(fcr->requestid));
                crp->crp_reqid = cnop[req].reqid;
                crp->crp_usropaque = cnop[req].opaque;
                cv_init(&crp->crp_cv, "crydev");
                crypto_dispatch(crp);
                cnop[req].status = 0;
                cv_destroy(&crp->crp_cv);
bail:
                if (cnop[req].status) {
                        DPRINTF("%zu status %d\n", req, cnop[req].status);
                        cse_free(cse);
                        if (cod)
                                cod_dtor(cod);
                        if (crp)
                                crypto_freereq(crp);
                        error = 0;
                }
        }
        return error;
}

static int
cryptodev_mkey(struct fcrypt *fcr, struct crypt_n_kop *kop, size_t count)
{
        struct cryptkop *krp = NULL;
        int error = EINVAL;
        size_t in, out, size, i, req;

        for (req = 0; req < count; req++) {
                if (kop[req].crk_iparams + kop[req].crk_oparams > CRK_MAXPARAM)
                        return SET_ERROR(EFBIG);

                in = kop[req].crk_iparams;
                out = kop[req].crk_oparams;
                switch (kop[req].crk_op) {
                case CRK_MOD_EXP:
                        if (in == 3 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_MOD_EXP_CRT:
                        if (in == 6 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_DSA_SIGN:
                        if (in == 5 && out == 2)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_DSA_VERIFY:
                        if (in == 7 && out == 0)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_DH_COMPUTE_KEY:
                        if (in == 3 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_MOD_ADD:
                        if (in == 3 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_MOD_ADDINV:
                        if (in == 2 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_MOD_SUB:
                        if (in == 3 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_MOD_MULT:
                        if (in == 3 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_MOD_MULTINV:
                        if (in == 2 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                case CRK_MOD:
                        if (in == 2 && out == 1)
                                break;
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                default:
                        kop[req].crk_status = SET_ERROR(EINVAL);
                        continue;
                }

                krp = crypto_kgetreq(1, PR_WAITOK);
                if (krp == NULL) {
                        /* limited by opencrypto.crypto_ret_kq.maxlen */
                        continue;
                }
                (void)memset(krp, 0, sizeof *krp);
                cv_init(&krp->krp_cv, "crykdev");
                krp->krp_op = kop[req].crk_op;
                krp->krp_status = kop[req].crk_status;
                krp->krp_iparams = kop[req].crk_iparams;
                krp->krp_oparams = kop[req].crk_oparams;
                krp->krp_status = 0;
                krp->krp_callback = cryptodevkey_mcb;
                (void)memcpy(krp->crk_param, kop[req].crk_param,
                    sizeof(kop[req].crk_param));

                krp->krp_flags = 0;

                for (i = 0; i < CRK_MAXPARAM; i++)
                        krp->krp_param[i].crp_nbits =
                            kop[req].crk_param[i].crp_nbits;
                for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
                        size = (krp->krp_param[i].crp_nbits + 7) / 8;
                        if (size == 0)
                                continue;
                        krp->krp_param[i].crp_p =
                            kmem_alloc(size, KM_SLEEP);
                        if (i >= krp->krp_iparams)
                                continue;
                        kop[req].crk_status =
                            copyin(kop[req].crk_param[i].crp_p,
                            krp->krp_param[i].crp_p, size);
                        if (kop[req].crk_status)
                                goto fail;
                }
                krp->fcrp = fcr;

                kop[req].crk_reqid = atomic_inc_32_nv(&(fcr->requestid));
                krp->krp_reqid = kop[req].crk_reqid;
                krp->krp_usropaque = kop[req].crk_opaque;

                crypto_kdispatch(krp);
                kop[req].crk_status = 0;
fail:
                if (kop[req].crk_status) {
                        if (krp) {
                                kop[req].crk_status = krp->krp_status;
                                for (i = 0; i < CRK_MAXPARAM; i++) {
                                        struct crparam *kp =
                                                &(krp->krp_param[i]);
                                        if (kp->crp_p) {
                                                size = (kp->crp_nbits + 7) / 8;
                                                KASSERT(size > 0);
                                                memset(kp->crp_p, 0, size);
                                                kmem_free(kp->crp_p, size);
                                        }
                                }
                                cv_destroy(&krp->krp_cv);
                                crypto_kfreereq(krp);
                        }
                }
                error = 0;
        }
        DPRINTF("error=0x%08x\n", error);
        return error;
}

int
cryptodev_session(struct fcrypt *fcr, struct session_op *sop)
{
        struct cryptoini cria, crie;
        struct cryptoini cric;          /* compressor */
        struct cryptoini *crihead = NULL;
        const struct enc_xform *txform = NULL;
        const struct auth_hash *thash = NULL;
        const struct comp_algo *tcomp = NULL;
        struct csession *cse;
        uint64_t sid;
        int error = 0;

        DPRINTF("cipher=%d, mac=%d\n", sop->cipher, sop->mac);

        /* XXX there must be a way to not embed the list of xforms here */
        switch (sop->cipher) {
        case 0:
                break;
        case CRYPTO_DES_CBC:
                txform = &enc_xform_des;
                break;
        case CRYPTO_3DES_CBC:
                txform = &enc_xform_3des;
                break;
        case CRYPTO_BLF_CBC:
                txform = &enc_xform_blf;
                break;
        case CRYPTO_CAST_CBC:
                txform = &enc_xform_cast5;
                break;
        case CRYPTO_SKIPJACK_CBC:
                txform = &enc_xform_skipjack;
                break;
        case CRYPTO_AES_CBC:
                txform = &enc_xform_aes;
                break;
        case CRYPTO_CAMELLIA_CBC:
                txform = &enc_xform_camellia;
                break;
        case CRYPTO_AES_CTR:
                txform = &enc_xform_aes_ctr;
                break;
        case CRYPTO_AES_GCM_16:
                txform = &enc_xform_aes_gcm;
                break;
        case CRYPTO_AES_GMAC:
                txform = &enc_xform_aes_gmac;
                break;
        case CRYPTO_NULL_CBC:
                txform = &enc_xform_null;
                break;
        case CRYPTO_ARC4:
                txform = &enc_xform_arc4;
                break;
        default:
                DPRINTF("Invalid cipher %d\n", sop->cipher);
                return SET_ERROR(EINVAL);
        }

        switch (sop->comp_alg) {
        case 0:
                break;
        case CRYPTO_DEFLATE_COMP:
                tcomp = &comp_algo_deflate;
                break;
        case CRYPTO_GZIP_COMP:
                tcomp = &comp_algo_gzip;
                DPRINTF("tcomp for GZIP\n");
                break;
        default:
                DPRINTF("Invalid compression alg %d\n", sop->comp_alg);
                return SET_ERROR(EINVAL);
        }

        switch (sop->mac) {
        case 0:
                break;
        case CRYPTO_MD5_HMAC:
                thash = &auth_hash_hmac_md5;
                break;
        case CRYPTO_SHA1_HMAC:
                thash = &auth_hash_hmac_sha1;
                break;
        case CRYPTO_MD5_HMAC_96:
                thash = &auth_hash_hmac_md5_96;
                break;
        case CRYPTO_SHA1_HMAC_96:
                thash = &auth_hash_hmac_sha1_96;
                break;
        case CRYPTO_SHA2_HMAC:
                /* XXX switching on key length seems questionable */
                if (sop->mackeylen == auth_hash_hmac_sha2_256.keysize) {
                        thash = &auth_hash_hmac_sha2_256;
                } else if (sop->mackeylen == auth_hash_hmac_sha2_384.keysize) {
                        thash = &auth_hash_hmac_sha2_384;
                } else if (sop->mackeylen == auth_hash_hmac_sha2_512.keysize) {
                        thash = &auth_hash_hmac_sha2_512;
                } else {
                        DPRINTF("Invalid mackeylen %d\n", sop->mackeylen);
                        return SET_ERROR(EINVAL);
                }
                break;
        case CRYPTO_SHA2_384_HMAC:
                thash = &auth_hash_hmac_sha2_384;
                break;
        case CRYPTO_SHA2_512_HMAC:
                thash = &auth_hash_hmac_sha2_512;
                break;
        case CRYPTO_RIPEMD160_HMAC:
                thash = &auth_hash_hmac_ripemd_160;
                break;
        case CRYPTO_RIPEMD160_HMAC_96:
                thash = &auth_hash_hmac_ripemd_160_96;
                break;
        case CRYPTO_MD5:
                thash = &auth_hash_md5;
                break;
        case CRYPTO_SHA1:
                thash = &auth_hash_sha1;
                break;
        case CRYPTO_AES_XCBC_MAC_96:
                thash = &auth_hash_aes_xcbc_mac_96;
                break;
        case CRYPTO_AES_128_GMAC:
                thash = &auth_hash_gmac_aes_128;
                break;
        case CRYPTO_AES_192_GMAC:
                thash = &auth_hash_gmac_aes_192;
                break;
        case CRYPTO_AES_256_GMAC:
                thash = &auth_hash_gmac_aes_256;
                break;
        case CRYPTO_NULL_HMAC:
                thash = &auth_hash_null;
                break;
        default:
                DPRINTF("Invalid mac %d\n", sop->mac);
                return SET_ERROR(EINVAL);
        }

        memset(&crie, 0, sizeof(crie));
        memset(&cria, 0, sizeof(cria));
        memset(&cric, 0, sizeof(cric));

        if (tcomp) {
                cric.cri_alg = tcomp->type;
                cric.cri_klen = 0;
                DPRINTF("tcomp->type = %d\n", tcomp->type);

                crihead = &cric;
                if (txform) {
                        cric.cri_next = &crie;
                } else if (thash) {
                        cric.cri_next = &cria;
                }
        }

        if (txform) {
                crie.cri_alg = txform->type;
                crie.cri_klen = sop->keylen * 8;
                if (sop->keylen > txform->maxkey ||
                    sop->keylen < txform->minkey) {
                        DPRINTF("keylen %d not in [%d,%d]\n",
                            sop->keylen, txform->minkey, txform->maxkey);
                        error = SET_ERROR(EINVAL);
                        goto bail;
                }

                crie.cri_key = malloc(crie.cri_klen / 8, M_XDATA, M_WAITOK);
                if ((error = copyin(sop->key, crie.cri_key, crie.cri_klen / 8)))
                        goto bail;
                if (!crihead) {
                        crihead = &crie;
                }
                if (thash)
                        crie.cri_next = &cria;
        }

        if (thash) {
                cria.cri_alg = thash->type;
                cria.cri_klen = sop->mackeylen * 8;
                if (sop->mackeylen != thash->keysize) {
                        DPRINTF("mackeylen %d != keysize %d\n",
                            sop->mackeylen, thash->keysize);
                        error = SET_ERROR(EINVAL);
                        goto bail;
                }
                if (cria.cri_klen) {
                        cria.cri_key = malloc(cria.cri_klen / 8, M_XDATA,
                            M_WAITOK);
                        if ((error = copyin(sop->mackey, cria.cri_key,
                            cria.cri_klen / 8))) {
                                goto bail;
                        }
                }
                if (!crihead) {
                        crihead = &cria;
                }
        }

        error = crypto_newsession(&sid, crihead, crypto_devallowsoft);
        if (!error) {
                DPRINTF("got session %d\n", (uint32_t)sid);
                cse = cse_create(fcr, sid, crie.cri_key, crie.cri_klen,
                    cria.cri_key, cria.cri_klen, (txform ? sop->cipher : 0), sop->mac,
                    (tcomp ? sop->comp_alg : 0), txform, thash, tcomp);
                if (cse != NULL) {
                        sop->ses = cse->ses;
                } else {
                        DPRINTF("csecreate failed\n");
                        crypto_freesession(sid);
                        error = SET_ERROR(EINVAL);
                }
        } else {
                DPRINTF("SIOCSESSION violates kernel parameters %d\n", error);
        }
bail:
        if (error) {
                if (crie.cri_key) {
                        memset(crie.cri_key, 0, crie.cri_klen / 8);
                        free(crie.cri_key, M_XDATA);
                }
                if (cria.cri_key) {
                        memset(cria.cri_key, 0, cria.cri_klen / 8);
                        free(cria.cri_key, M_XDATA);
                }
        }
        return error;
}

int
cryptodev_msession(struct fcrypt *fcr, struct session_n_op *sn_ops,
    size_t count)
{
        size_t i;

        for (i = 0; i < count; i++, sn_ops++) {
                struct session_op s_op;
                s_op.cipher =           sn_ops->cipher;
                s_op.mac =              sn_ops->mac;
                s_op.comp_alg =         sn_ops->comp_alg;
                s_op.keylen =           sn_ops->keylen;
                s_op.key =              sn_ops->key;
                s_op.mackeylen =        sn_ops->mackeylen;
                s_op.mackey =           sn_ops->mackey;
                s_op.ses =              ~0;

                sn_ops->status = cryptodev_session(fcr, &s_op);
                if (sn_ops->status) {
                        DPRINTF("%zu: create failed %d\n", i, sn_ops->status);
                }
                sn_ops->ses =           s_op.ses;
        }

        return 0;
}

static void
cryptodev_msessionfin(struct fcrypt *fcr, size_t count, uint32_t *sesid)
{
        mutex_enter(&fcr->lock);
        for (size_t req = 0; req < count; req++) {
                // XXX: What to do if busy?
                cse_delete(fcr, sesid[req]);
        }
        mutex_exit(&fcr->lock);
}

/*
 * collect as many completed requests as are available, or count completed
 * requests, whichever is less.
 * return the number of requests.
 */
static int
cryptodev_getmstatus(struct fcrypt *fcr, struct crypt_result *crypt_res,
    int count)
{
        struct cryptop *crp = NULL;
        struct cryptkop *krp = NULL;
        int i, size, req = 0;
        int completed=0;

        /* On queue so nobody else can grab them
         * and copyout can be delayed-- no locking */
        TAILQ_HEAD(, cryptop) crp_delfree_q =
                TAILQ_HEAD_INITIALIZER(crp_delfree_q);
        TAILQ_HEAD(, cryptkop) krp_delfree_q =
                TAILQ_HEAD_INITIALIZER(krp_delfree_q);

        /* at this point we do not know which response user is requesting for
         * (symmetric or asymmetric) so we copyout one from each i.e if the
         * count is 2 then 1 from symmetric and 1 from asymmetric queue and
         * if 3 then 2 symmetric and 1 asymmetric and so on */

        /* pull off a list of requests while protected from changes */
        mutex_enter(&fcr->lock);
        while (req < count) {
                crp = TAILQ_FIRST(&fcr->crp_ret_mq);
                if (crp) {
                        TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next);
                        TAILQ_INSERT_TAIL(&crp_delfree_q, crp, crp_next);
                        crypt_res[req].status = 0;
                        req++;
                }
                if (req < count) {
                        crypt_res[req].status = 0;
                        krp = TAILQ_FIRST(&fcr->crp_ret_mkq);
                        if (krp) {
                                TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next);
                                TAILQ_INSERT_TAIL(&krp_delfree_q, krp, krp_next);
                        req++;
                        }
                }
        }
        mutex_exit(&fcr->lock);

        /* now do all the work outside the mutex */
        for(req=0; req < count ;) {
                crp = TAILQ_FIRST(&crp_delfree_q);
                if (crp) {
                        struct cryptop_data *cod = crp->crp_opaque;
                        if (crypt_res[req].status != 0) {
                                /* cse_find failed during collection */
                                goto bail;
                        }
                        crypt_res[req].reqid = crp->crp_reqid;
                        crypt_res[req].opaque = crp->crp_usropaque;
                        completed++;

                        if (crp->crp_etype != 0) {
                                crypt_res[req].status = crp->crp_etype;
                                goto bail;
                        }

                        if (crp->dst && (crypt_res[req].status =
                            copyout(cod->uio.uio_iov[0].iov_base, crp->dst,
                            crp->len)))
                                goto bail;

                        if (crp->mac && (crypt_res[req].status =
                            copyout(crp->crp_mac, crp->mac,
                            cod->cse->thash->authsize)))
                                goto bail;

bail:
                        TAILQ_REMOVE(&crp_delfree_q, crp, crp_next);
                        cse_free(cod->cse);
                        cod_dtor(cod);
                        crypto_freereq(crp);
                        req++;
                }

                if (req < count) {
                        krp = TAILQ_FIRST(&krp_delfree_q);
                        if (krp) {
                                crypt_res[req].reqid = krp->krp_reqid;
                                crypt_res[req].opaque = krp->krp_usropaque;
                                completed++;
                                if (krp->krp_status != 0) {
                                        DPRINTF("krp->krp_status 0x%08x\n",
                                            krp->krp_status);
                                        crypt_res[req].status = krp->krp_status;
                                        goto fail;
                                }

                                for (i = krp->krp_iparams; i < krp->krp_iparams
                                    + krp->krp_oparams; i++) {
                                        size = (krp->krp_param[i].crp_nbits
                                            + 7) / 8;
                                        if (size == 0)
                                                continue;
                                        crypt_res[req].status = copyout
                                            (krp->krp_param[i].crp_p,
                                            krp->crk_param[i].crp_p, size);
                                        if (crypt_res[req].status) {
                                                DPRINTF("copyout oparam %d failed, "
                                                    "error=%d\n",
                                                    i - krp->krp_iparams,
                                                    crypt_res[req].status);
                                                goto fail;
                                        }
                                }
fail:
                                TAILQ_REMOVE(&krp_delfree_q, krp, krp_next);
                                /* not sure what to do for this */
                                /* kop[req].crk_status = krp->krp_status; */
                                for (i = 0; i < CRK_MAXPARAM; i++) {
                                        struct crparam *kp = &(krp->krp_param[i]);
                                        if (kp->crp_p) {
                                                size = (kp->crp_nbits + 7) / 8;
                                                KASSERT(size > 0);
                                                (void)memset(kp->crp_p, 0, size);
                                                kmem_free(kp->crp_p, size);
                                        }
                                }
                                cv_destroy(&krp->krp_cv);
                                crypto_kfreereq(krp);
                                req++;
                        }
                }
        }

        return completed;
}

static int
cryptodev_getstatus (struct fcrypt *fcr, struct crypt_result *crypt_res)
{
        struct cryptop *crp = NULL, *cnext;
        struct cryptkop *krp = NULL, *knext;
        int i, size, req = 0;

        mutex_enter(&fcr->lock);
        /* Here we dont know for which request the user is requesting the
         * response so checking in both the queues */
        TAILQ_FOREACH_SAFE(crp, &fcr->crp_ret_mq, crp_next, cnext) {
                if(crp && (crp->crp_reqid == crypt_res->reqid)) {
                        struct cryptop_data *cod = crp->crp_opaque;
                        crypt_res->opaque = crp->crp_usropaque;
                        if (crp->crp_etype != 0) {
                                crypt_res->status = crp->crp_etype;
                                goto bail;
                        }

                        if (crp->dst && (crypt_res->status =
                            copyout(cod->uio.uio_iov[0].iov_base,
                            crp->dst, crp->len)))
                                goto bail;

                        if (crp->mac && (crypt_res->status =
                            copyout(crp->crp_mac, crp->mac,
                            cod->cse->thash->authsize)))
                                goto bail;
bail:
                        TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next);
                        mutex_exit(&fcr->lock);
                        cse_free(cod->cse);
                        cod_dtor(cod);
                        crypto_freereq(crp);
                        return 0;
                }
        }

        TAILQ_FOREACH_SAFE(krp, &fcr->crp_ret_mkq, krp_next, knext) {
                if(krp && (krp->krp_reqid == crypt_res->reqid)) {
                        crypt_res[req].opaque = krp->krp_usropaque;
                        if (krp->krp_status != 0) {
                                DPRINTF("krp->krp_status 0x%08x\n",
                                    krp->krp_status);
                                crypt_res[req].status = krp->krp_status;
                                goto fail;
                        }

                        for (i = krp->krp_iparams; i < krp->krp_iparams +
                            krp->krp_oparams; i++) {
                                size = (krp->krp_param[i].crp_nbits + 7) / 8;
                                if (size == 0)
                                        continue;
                                crypt_res[req].status = copyout(
                                    krp->krp_param[i].crp_p,
                                    krp->crk_param[i].crp_p, size);
                                if (crypt_res[req].status) {
                                        DPRINTF("copyout oparam "
                                            "%d failed, error=%d\n",
                                            i - krp->krp_iparams,
                                            crypt_res[req].status);
                                        goto fail;
                                }
                        }
fail:
                        TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next);
                        mutex_exit(&fcr->lock);
                        /* not sure what to do for this */
                        /* kop[req].crk_status = krp->krp_status; */
                        for (i = 0; i < CRK_MAXPARAM; i++) {
                                struct crparam *kp = &(krp->krp_param[i]);
                                if (kp->crp_p) {
                                        size = (kp->crp_nbits + 7) / 8;
                                        KASSERT(size > 0);
                                        memset(kp->crp_p, 0, size);
                                        kmem_free(kp->crp_p, size);
                                }
                        }
                        cv_destroy(&krp->krp_cv);
                        crypto_kfreereq(krp);
                        return 0;
                }
        }
        mutex_exit(&fcr->lock);
        return SET_ERROR(EINPROGRESS);
}

static int
cryptof_stat(struct file *fp, struct stat *st)
{
        struct fcrypt *fcr = fp->f_fcrypt;

        (void)memset(st, 0, sizeof(*st));

        mutex_enter(&fcr->lock);
        st->st_dev = makedev(cdevsw_lookup_major(&crypto_cdevsw), fcr->sesn);
        st->st_atimespec = fcr->atime;
        st->st_mtimespec = fcr->mtime;
        st->st_ctimespec = st->st_birthtimespec = fcr->btime;
        st->st_uid = kauth_cred_geteuid(fp->f_cred);
        st->st_gid = kauth_cred_getegid(fp->f_cred);
        mutex_exit(&fcr->lock);

        return 0;
}

static int
cryptof_poll(struct file *fp, int events)
{
        struct fcrypt *fcr = fp->f_fcrypt;
        int revents = 0;

        if (!(events & (POLLIN | POLLRDNORM))) {
                /* only support read and POLLIN */
                return 0;
        }

        mutex_enter(&fcr->lock);
        if (TAILQ_EMPTY(&fcr->crp_ret_mq) && TAILQ_EMPTY(&fcr->crp_ret_mkq)) {
                /* no completed requests pending, save the poll for later */
                selrecord(curlwp, &fcr->sinfo);
        } else {
                /* let the app(s) know that there are completed requests */
                revents = events & (POLLIN | POLLRDNORM);
        }
        mutex_exit(&fcr->lock);

        return revents;
}

static void
cryptodev_init(void)
{

        mutex_init(&cryptodev_mtx, MUTEX_DEFAULT, IPL_NONE);

        pool_init(&fcrpl, sizeof(struct fcrypt), 0, 0, 0, "fcrpl",
            NULL, IPL_NONE);
        pool_init(&csepl, sizeof(struct csession), 0, 0, 0, "csepl",
            NULL, IPL_NONE);

        /*
         * Preallocate space for 64 users, with 5 sessions each.
         * (consider that a TLS protocol session requires at least
         * 3DES, MD5, and SHA1 (both hashes are used in the PRF) for
         * the negotiation, plus HMAC_SHA1 for the actual SSL records,
         * consuming one session here for each algorithm.
         */
        pool_prime(&fcrpl, 64);
        pool_prime(&csepl, 64 * 5);
}

static void
cryptodev_fini(void)
{

        KASSERTMSG(crypto_refcount == 0,
            "crypto_refcount=%d", crypto_refcount);

        pool_destroy(&csepl);
        pool_destroy(&fcrpl);

        mutex_destroy(&cryptodev_mtx);
}

/*
 * Pseudo-device initialization routine for /dev/crypto
 */
void
cryptoattach(int num)
{
#ifndef _MODULE
        crypto_init();
        cryptodev_init();
#endif
}

MODULE(MODULE_CLASS_DRIVER, crypto, "opencrypto");

static int
crypto_modcmd(modcmd_t cmd, void *arg)
{
#ifdef _MODULE
        int error = 0;
        devmajor_t cmajor = NODEVMAJOR, bmajor = NODEVMAJOR;
#endif

        switch (cmd) {
        case MODULE_CMD_INIT:
#ifdef _MODULE
                cryptodev_init();
                error = devsw_attach("crypto", NULL, &bmajor,
                    &crypto_cdevsw, &cmajor);
                if (error) {
                        aprint_error("crypto: unable to register devsw,"
                            " error %d\n", error);
                        cryptodev_fini();
                        return error;
                }
#endif
                return 0;
        case MODULE_CMD_FINI:
                /*
                 * Can't unload safely because there is no way to wait
                 * for all files with a given struct fileops to be
                 * closed.
                 */
                return SET_ERROR(ENOTTY);

                mutex_enter(&cryptodev_mtx);
                if (crypto_refcount != 0) {
                        mutex_exit(&cryptodev_mtx);
                        return SET_ERROR(EBUSY);
                }
                crypto_detaching = true;
                mutex_exit(&cryptodev_mtx);
                devsw_detach(NULL, &crypto_cdevsw);
                cryptodev_fini();
                return 0;
        default:
                return SET_ERROR(ENOTTY);
        }
}