/*      $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */

/*-
 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
 *
 * This code was written by Angelos D. Keromytis in Athens, Greece, in
 * February 2000. Network Security Technologies Inc. (NSTI) kindly
 * supported the development of this code.
 *
 * Copyright (c) 2000, 2001 Angelos D. Keromytis
 * Copyright (c) 2014-2021 The FreeBSD Foundation
 * All rights reserved.
 *
 * Portions of this software were developed by John-Mark Gurney
 * under sponsorship of the FreeBSD Foundation and
 * Rubicon Communications, LLC (Netgate).
 *
 * Portions of this software were developed by Ararat River
 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
 *
 * Permission to use, copy, and modify this software with or without fee
 * is hereby granted, provided that this entire notice is included in
 * all source code copies of any software which is or includes a copy or
 * modification of this software.
 *
 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
 * PURPOSE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/sysctl.h>
#include <sys/errno.h>
#include <sys/random.h>
#include <sys/kernel.h>
#include <sys/uio.h>
#include <sys/endian.h>
#include <sys/limits.h>

#include <crypto/sha1.h>
#include <opencrypto/rmd160.h>

#include <opencrypto/cryptodev.h>
#include <opencrypto/xform.h>

#include <sys/kobj.h>
#include <sys/bus.h>
#include "cryptodev_if.h"

struct swcr_auth {
        void            *sw_ictx;
        void            *sw_octx;
        const struct auth_hash *sw_axf;
        uint16_t        sw_mlen;
        bool            sw_hmac;
};

struct swcr_encdec {
        void            *sw_ctx;
        const struct enc_xform *sw_exf;
};

struct swcr_compdec {
        const struct comp_algo *sw_cxf;
};

struct swcr_session {
        int     (*swcr_process)(const struct swcr_session *, struct cryptop *);

        struct swcr_auth swcr_auth;
        struct swcr_encdec swcr_encdec;
        struct swcr_compdec swcr_compdec;
};

static  int32_t swcr_id;

static  void swcr_freesession(device_t dev, crypto_session_t cses);

/* Used for CRYPTO_NULL_CBC. */
static int
swcr_null(const struct swcr_session *ses, struct cryptop *crp)
{

        return (0);
}

/*
 * Apply a symmetric encryption/decryption algorithm.
 */
static int
swcr_encdec(const struct swcr_session *ses, struct cryptop *crp)
{
        unsigned char blk[EALG_MAX_BLOCK_LEN];
        const struct crypto_session_params *csp;
        const struct enc_xform *exf;
        const struct swcr_encdec *sw;
        void *ctx;
        size_t inlen, outlen, todo;
        int blksz, resid;
        struct crypto_buffer_cursor cc_in, cc_out;
        const unsigned char *inblk;
        unsigned char *outblk;
        int error;
        bool encrypting;

        error = 0;

        sw = &ses->swcr_encdec;
        exf = sw->sw_exf;
        csp = crypto_get_params(crp->crp_session);

        if (exf->native_blocksize == 0) {
                /* Check for non-padded data */
                if ((crp->crp_payload_length % exf->blocksize) != 0)
                        return (EINVAL);

                blksz = exf->blocksize;
        } else
                blksz = exf->native_blocksize;

        if (exf == &enc_xform_aes_icm &&
            (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
                return (EINVAL);

        ctx = __builtin_alloca(exf->ctxsize);
        if (crp->crp_cipher_key != NULL) {
                error = exf->setkey(ctx, crp->crp_cipher_key,
                    csp->csp_cipher_klen);
                if (error)
                        return (error);
        } else
                memcpy(ctx, sw->sw_ctx, exf->ctxsize);

        crypto_read_iv(crp, blk);
        exf->reinit(ctx, blk, csp->csp_ivlen);

        crypto_cursor_init(&cc_in, &crp->crp_buf);
        crypto_cursor_advance(&cc_in, crp->crp_payload_start);
        if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
                crypto_cursor_init(&cc_out, &crp->crp_obuf);
                crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
        } else
                cc_out = cc_in;

        encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);

        /*
         * Loop through encrypting blocks.  'inlen' is the remaining
         * length of the current segment in the input buffer.
         * 'outlen' is the remaining length of current segment in the
         * output buffer.
         */
        inlen = outlen = 0;
        for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
                if (inlen == 0)
                        inblk = crypto_cursor_segment(&cc_in, &inlen);
                if (outlen == 0)
                        outblk = crypto_cursor_segment(&cc_out, &outlen);

                /*
                 * If the current block is not contained within the
                 * current input/output segment, use 'blk' as a local
                 * buffer.
                 */
                if (inlen < blksz) {
                        crypto_cursor_copydata(&cc_in, blksz, blk);
                        inblk = blk;
                        inlen = blksz;
                }
                if (outlen < blksz) {
                        outblk = blk;
                        outlen = blksz;
                }

                todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz);

                if (encrypting)
                        exf->encrypt_multi(ctx, inblk, outblk, todo);
                else
                        exf->decrypt_multi(ctx, inblk, outblk, todo);

                if (inblk == blk) {
                        inblk = crypto_cursor_segment(&cc_in, &inlen);
                } else {
                        crypto_cursor_advance(&cc_in, todo);
                        inlen -= todo;
                        inblk += todo;
                }

                if (outblk == blk) {
                        crypto_cursor_copyback(&cc_out, blksz, blk);
                        outblk = crypto_cursor_segment(&cc_out, &outlen);
                } else {
                        crypto_cursor_advance(&cc_out, todo);
                        outlen -= todo;
                        outblk += todo;
                }
        }

        /* Handle trailing partial block for stream ciphers. */
        if (resid > 0) {
                KASSERT(exf->native_blocksize != 0,
                    ("%s: partial block of %d bytes for cipher %s",
                    __func__, resid, exf->name));
                KASSERT(resid < blksz, ("%s: partial block too big", __func__));

                inblk = crypto_cursor_segment(&cc_in, &inlen);
                outblk = crypto_cursor_segment(&cc_out, &outlen);
                if (inlen < resid) {
                        crypto_cursor_copydata(&cc_in, resid, blk);
                        inblk = blk;
                }
                if (outlen < resid)
                        outblk = blk;
                if (encrypting)
                        exf->encrypt_last(ctx, inblk, outblk,
                            resid);
                else
                        exf->decrypt_last(ctx, inblk, outblk,
                            resid);
                if (outlen < resid)
                        crypto_cursor_copyback(&cc_out, resid, blk);
        }

        explicit_bzero(ctx, exf->ctxsize);
        explicit_bzero(blk, sizeof(blk));
        return (0);
}

/*
 * Compute or verify hash.
 */
static int
swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp)
{
        struct {
                union authctx ctx;
                u_char aalg[HASH_MAX_LEN];
                u_char uaalg[HASH_MAX_LEN];
        } s;
        const struct crypto_session_params *csp;
        const struct swcr_auth *sw;
        const struct auth_hash *axf;
        int err;

        sw = &ses->swcr_auth;

        axf = sw->sw_axf;

        csp = crypto_get_params(crp->crp_session);
        if (crp->crp_auth_key != NULL) {
                if (sw->sw_hmac) {
                        hmac_init_ipad(axf, crp->crp_auth_key,
                            csp->csp_auth_klen, &s.ctx);
                } else {
                        axf->Init(&s.ctx);
                        axf->Setkey(&s.ctx, crp->crp_auth_key,
                            csp->csp_auth_klen);
                }
        } else
                memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize);

        if (crp->crp_aad != NULL)
                err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length);
        else
                err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
                    axf->Update, &s.ctx);
        if (err)
                goto out;

        if (CRYPTO_HAS_OUTPUT_BUFFER(crp) &&
            CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
                err = crypto_apply_buf(&crp->crp_obuf,
                    crp->crp_payload_output_start, crp->crp_payload_length,
                    axf->Update, &s.ctx);
        else
                err = crypto_apply(crp, crp->crp_payload_start,
                    crp->crp_payload_length, axf->Update, &s.ctx);
        if (err)
                goto out;

        if (csp->csp_flags & CSP_F_ESN)
                axf->Update(&s.ctx, crp->crp_esn, 4);

        axf->Final(s.aalg, &s.ctx);
        if (sw->sw_hmac) {
                if (crp->crp_auth_key != NULL)
                        hmac_init_opad(axf, crp->crp_auth_key,
                            csp->csp_auth_klen, &s.ctx);
                else
                        memcpy(&s.ctx, sw->sw_octx, axf->ctxsize);
                axf->Update(&s.ctx, s.aalg, axf->hashsize);
                axf->Final(s.aalg, &s.ctx);
        }

        if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
                crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg);
                if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0)
                        err = EBADMSG;
        } else {
                /* Inject the authentication data */
                crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg);
        }
out:
        explicit_bzero(&s, sizeof(s));
        return (err);
}

CTASSERT(INT_MAX <= (1ll<<39) - 256);   /* GCM: plain text < 2^39-256 */
CTASSERT(INT_MAX <= (uint64_t)-1);      /* GCM: associated data <= 2^64-1 */

static int
swcr_gmac(const struct swcr_session *ses, struct cryptop *crp)
{
        struct {
                union authctx ctx;
                uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))];
                u_char tag[GMAC_DIGEST_LEN];
                u_char tag2[GMAC_DIGEST_LEN];
        } s;
        u_char *blk = (u_char *)s.blkbuf;
        struct crypto_buffer_cursor cc;
        const u_char *inblk;
        const struct swcr_auth *swa;
        const struct auth_hash *axf;
        uint32_t *blkp;
        size_t len;
        int blksz, error, ivlen, resid;

        swa = &ses->swcr_auth;
        axf = swa->sw_axf;
        blksz = GMAC_BLOCK_LEN;
        KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch",
            __func__));

        if (crp->crp_auth_key != NULL) {
                axf->Init(&s.ctx);
                axf->Setkey(&s.ctx, crp->crp_auth_key,
                    crypto_get_params(crp->crp_session)->csp_auth_klen);
        } else
                memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize);

        /* Initialize the IV */
        ivlen = AES_GCM_IV_LEN;
        crypto_read_iv(crp, blk);

        axf->Reinit(&s.ctx, blk, ivlen);
        crypto_cursor_init(&cc, &crp->crp_buf);
        crypto_cursor_advance(&cc, crp->crp_payload_start);
        for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) {
                inblk = crypto_cursor_segment(&cc, &len);
                if (len >= blksz) {
                        len = rounddown(MIN(len, resid), blksz);
                        crypto_cursor_advance(&cc, len);
                } else {
                        len = blksz;
                        crypto_cursor_copydata(&cc, len, blk);
                        inblk = blk;
                }
                axf->Update(&s.ctx, inblk, len);
        }
        if (resid > 0) {
                memset(blk, 0, blksz);
                crypto_cursor_copydata(&cc, resid, blk);
                axf->Update(&s.ctx, blk, blksz);
        }

        /* length block */
        memset(blk, 0, blksz);
        blkp = (uint32_t *)blk + 1;
        *blkp = htobe32(crp->crp_payload_length * 8);
        axf->Update(&s.ctx, blk, blksz);

        /* Finalize MAC */
        axf->Final(s.tag, &s.ctx);

        error = 0;
        if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
                crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag2);
                if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0)
                        error = EBADMSG;
        } else {
                /* Inject the authentication data */
                crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag);
        }
        explicit_bzero(&s, sizeof(s));
        return (error);
}

static int
swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
{
        struct {
                uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))];
                u_char tag[GMAC_DIGEST_LEN];
                u_char tag2[GMAC_DIGEST_LEN];
        } s;
        u_char *blk = (u_char *)s.blkbuf;
        struct crypto_buffer_cursor cc_in, cc_out;
        const u_char *inblk;
        u_char *outblk;
        size_t inlen, outlen, todo;
        const struct swcr_auth *swa;
        const struct swcr_encdec *swe;
        const struct enc_xform *exf;
        void *ctx;
        uint32_t *blkp;
        int blksz, error, ivlen, r, resid;

        swa = &ses->swcr_auth;
        swe = &ses->swcr_encdec;
        exf = swe->sw_exf;
        blksz = GMAC_BLOCK_LEN;
        KASSERT(blksz == exf->native_blocksize,
            ("%s: blocksize mismatch", __func__));

        if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
                return (EINVAL);

        ivlen = AES_GCM_IV_LEN;

        ctx = __builtin_alloca(exf->ctxsize);
        if (crp->crp_cipher_key != NULL)
                exf->setkey(ctx, crp->crp_cipher_key,
                    crypto_get_params(crp->crp_session)->csp_cipher_klen);
        else
                memcpy(ctx, swe->sw_ctx, exf->ctxsize);
        exf->reinit(ctx, crp->crp_iv, ivlen);

        /* Supply MAC with AAD */
        if (crp->crp_aad != NULL) {
                inlen = rounddown2(crp->crp_aad_length, blksz);
                if (inlen != 0)
                        exf->update(ctx, crp->crp_aad, inlen);
                if (crp->crp_aad_length != inlen) {
                        memset(blk, 0, blksz);
                        memcpy(blk, (char *)crp->crp_aad + inlen,
                            crp->crp_aad_length - inlen);
                        exf->update(ctx, blk, blksz);
                }
        } else {
                crypto_cursor_init(&cc_in, &crp->crp_buf);
                crypto_cursor_advance(&cc_in, crp->crp_aad_start);
                for (resid = crp->crp_aad_length; resid >= blksz;
                     resid -= inlen) {
                        inblk = crypto_cursor_segment(&cc_in, &inlen);
                        if (inlen >= blksz) {
                                inlen = rounddown2(MIN(inlen, resid), blksz);
                                crypto_cursor_advance(&cc_in, inlen);
                        } else {
                                inlen = blksz;
                                crypto_cursor_copydata(&cc_in, inlen, blk);
                                inblk = blk;
                        }
                        exf->update(ctx, inblk, inlen);
                }
                if (resid > 0) {
                        memset(blk, 0, blksz);
                        crypto_cursor_copydata(&cc_in, resid, blk);
                        exf->update(ctx, blk, blksz);
                }
        }

        /* Do encryption with MAC */
        crypto_cursor_init(&cc_in, &crp->crp_buf);
        crypto_cursor_advance(&cc_in, crp->crp_payload_start);
        if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
                crypto_cursor_init(&cc_out, &crp->crp_obuf);
                crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
        } else
                cc_out = cc_in;

        inlen = outlen = 0;
        for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
                if (inlen == 0)
                        inblk = crypto_cursor_segment(&cc_in, &inlen);
                if (outlen == 0)
                        outblk = crypto_cursor_segment(&cc_out, &outlen);

                if (inlen < blksz) {
                        crypto_cursor_copydata(&cc_in, blksz, blk);
                        inblk = blk;
                        inlen = blksz;
                }

                if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                        if (outlen < blksz) {
                                outblk = blk;
                                outlen = blksz;
                        }

                        todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
                            blksz);

                        exf->encrypt_multi(ctx, inblk, outblk, todo);
                        exf->update(ctx, outblk, todo);

                        if (outblk == blk) {
                                crypto_cursor_copyback(&cc_out, blksz, blk);
                                outblk = crypto_cursor_segment(&cc_out, &outlen);
                        } else {
                                crypto_cursor_advance(&cc_out, todo);
                                outlen -= todo;
                                outblk += todo;
                        }
                } else {
                        todo = rounddown2(MIN(resid, inlen), blksz);
                        exf->update(ctx, inblk, todo);
                }

                if (inblk == blk) {
                        inblk = crypto_cursor_segment(&cc_in, &inlen);
                } else {
                        crypto_cursor_advance(&cc_in, todo);
                        inlen -= todo;
                        inblk += todo;
                }
        }
        if (resid > 0) {
                crypto_cursor_copydata(&cc_in, resid, blk);
                if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                        exf->encrypt_last(ctx, blk, blk, resid);
                        crypto_cursor_copyback(&cc_out, resid, blk);
                }
                exf->update(ctx, blk, resid);
        }

        /* length block */
        memset(blk, 0, blksz);
        blkp = (uint32_t *)blk + 1;
        *blkp = htobe32(crp->crp_aad_length * 8);
        blkp = (uint32_t *)blk + 3;
        *blkp = htobe32(crp->crp_payload_length * 8);
        exf->update(ctx, blk, blksz);

        /* Finalize MAC */
        exf->final(s.tag, ctx);

        /* Validate tag */
        error = 0;
        if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag2);
                r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen);
                if (r != 0) {
                        error = EBADMSG;
                        goto out;
                }

                /* tag matches, decrypt data */
                crypto_cursor_init(&cc_in, &crp->crp_buf);
                crypto_cursor_advance(&cc_in, crp->crp_payload_start);

                inlen = 0;
                for (resid = crp->crp_payload_length; resid > blksz;
                     resid -= todo) {
                        if (inlen == 0)
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        if (outlen == 0)
                                outblk = crypto_cursor_segment(&cc_out, &outlen);
                        if (inlen < blksz) {
                                crypto_cursor_copydata(&cc_in, blksz, blk);
                                inblk = blk;
                                inlen = blksz;
                        }
                        if (outlen < blksz) {
                                outblk = blk;
                                outlen = blksz;
                        }

                        todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
                            blksz);

                        exf->decrypt_multi(ctx, inblk, outblk, todo);

                        if (inblk == blk) {
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        } else {
                                crypto_cursor_advance(&cc_in, todo);
                                inlen -= todo;
                                inblk += todo;
                        }

                        if (outblk == blk) {
                                crypto_cursor_copyback(&cc_out, blksz, blk);
                                outblk = crypto_cursor_segment(&cc_out,
                                    &outlen);
                        } else {
                                crypto_cursor_advance(&cc_out, todo);
                                outlen -= todo;
                                outblk += todo;
                        }
                }
                if (resid > 0) {
                        crypto_cursor_copydata(&cc_in, resid, blk);
                        exf->decrypt_last(ctx, blk, blk, resid);
                        crypto_cursor_copyback(&cc_out, resid, blk);
                }
        } else {
                /* Inject the authentication data */
                crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag);
        }

out:
        explicit_bzero(ctx, exf->ctxsize);
        explicit_bzero(&s, sizeof(s));

        return (error);
}

static void
build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length,
    u_int data_length, u_int tag_length, uint8_t *b0)
{
        uint8_t *bp;
        uint8_t flags, L;

        KASSERT(nonce_length >= 7 && nonce_length <= 13,
            ("nonce_length must be between 7 and 13 bytes"));

        /*
         * Need to determine the L field value.  This is the number of
         * bytes needed to specify the length of the message; the length
         * is whatever is left in the 16 bytes after specifying flags and
         * the nonce.
         */
        L = 15 - nonce_length;

        flags = ((aad_length > 0) << 6) +
            (((tag_length - 2) / 2) << 3) +
            L - 1;

        /*
         * Now we need to set up the first block, which has flags, nonce,
         * and the message length.
         */
        b0[0] = flags;
        memcpy(b0 + 1, nonce, nonce_length);
        bp = b0 + 1 + nonce_length;

        /* Need to copy L' [aka L-1] bytes of data_length */
        for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) {
                *dst = data_length;
                data_length >>= 8;
        }
}

/* NB: OCF only supports AAD lengths < 2^32. */
static int
build_ccm_aad_length(u_int aad_length, uint8_t *blk)
{
        if (aad_length < ((1 << 16) - (1 << 8))) {
                be16enc(blk, aad_length);
                return (sizeof(uint16_t));
        } else {
                blk[0] = 0xff;
                blk[1] = 0xfe;
                be32enc(blk + 2, aad_length);
                return (2 + sizeof(uint32_t));
        }
}

static int
swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp)
{
        struct {
                union authctx ctx;
                u_char blk[CCM_CBC_BLOCK_LEN];
                u_char tag[AES_CBC_MAC_HASH_LEN];
                u_char tag2[AES_CBC_MAC_HASH_LEN];
        } s;
        const struct crypto_session_params *csp;
        const struct swcr_auth *swa;
        const struct auth_hash *axf;
        int error, ivlen, len;

        csp = crypto_get_params(crp->crp_session);
        swa = &ses->swcr_auth;
        axf = swa->sw_axf;

        if (crp->crp_auth_key != NULL) {
                axf->Init(&s.ctx);
                axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen);
        } else
                memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize);

        /* Initialize the IV */
        ivlen = csp->csp_ivlen;

        /* Supply MAC with IV */
        axf->Reinit(&s.ctx, crp->crp_iv, ivlen);

        /* Supply MAC with b0. */
        build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0,
            swa->sw_mlen, s.blk);
        axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN);

        len = build_ccm_aad_length(crp->crp_payload_length, s.blk);
        axf->Update(&s.ctx, s.blk, len);

        crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length,
            axf->Update, &s.ctx);

        /* Finalize MAC */
        axf->Final(s.tag, &s.ctx);

        error = 0;
        if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
                crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag2);
                if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0)
                        error = EBADMSG;
        } else {
                /* Inject the authentication data */
                crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag);
        }
        explicit_bzero(&s, sizeof(s));
        return (error);
}

static int
swcr_ccm(const struct swcr_session *ses, struct cryptop *crp)
{
        const struct crypto_session_params *csp;
        struct {
                uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))];
                u_char tag[AES_CBC_MAC_HASH_LEN];
                u_char tag2[AES_CBC_MAC_HASH_LEN];
        } s;
        u_char *blk = (u_char *)s.blkbuf;
        struct crypto_buffer_cursor cc_in, cc_out;
        const u_char *inblk;
        u_char *outblk;
        size_t inlen, outlen, todo;
        const struct swcr_auth *swa;
        const struct swcr_encdec *swe;
        const struct enc_xform *exf;
        void *ctx;
        size_t len;
        int blksz, error, ivlen, r, resid;

        csp = crypto_get_params(crp->crp_session);
        swa = &ses->swcr_auth;
        swe = &ses->swcr_encdec;
        exf = swe->sw_exf;
        blksz = AES_BLOCK_LEN;
        KASSERT(blksz == exf->native_blocksize,
            ("%s: blocksize mismatch", __func__));

        if (crp->crp_payload_length > ccm_max_payload_length(csp))
                return (EMSGSIZE);

        if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
                return (EINVAL);

        ivlen = csp->csp_ivlen;

        ctx = __builtin_alloca(exf->ctxsize);
        if (crp->crp_cipher_key != NULL)
                exf->setkey(ctx, crp->crp_cipher_key,
                    crypto_get_params(crp->crp_session)->csp_cipher_klen);
        else
                memcpy(ctx, swe->sw_ctx, exf->ctxsize);
        exf->reinit(ctx, crp->crp_iv, ivlen);

        /* Supply MAC with b0. */
        _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN,
            "blkbuf too small for b0");
        build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length,
            crp->crp_payload_length, swa->sw_mlen, blk);
        exf->update(ctx, blk, CCM_CBC_BLOCK_LEN);

        /* Supply MAC with AAD */
        if (crp->crp_aad_length != 0) {
                len = build_ccm_aad_length(crp->crp_aad_length, blk);
                exf->update(ctx, blk, len);
                if (crp->crp_aad != NULL)
                        exf->update(ctx, crp->crp_aad, crp->crp_aad_length);
                else
                        crypto_apply(crp, crp->crp_aad_start,
                            crp->crp_aad_length, exf->update, ctx);

                /* Pad the AAD (including length field) to a full block. */
                len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN;
                if (len != 0) {
                        len = CCM_CBC_BLOCK_LEN - len;
                        memset(blk, 0, CCM_CBC_BLOCK_LEN);
                        exf->update(ctx, blk, len);
                }
        }

        /* Do encryption/decryption with MAC */
        crypto_cursor_init(&cc_in, &crp->crp_buf);
        crypto_cursor_advance(&cc_in, crp->crp_payload_start);
        if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
                crypto_cursor_init(&cc_out, &crp->crp_obuf);
                crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
        } else
                cc_out = cc_in;

        inlen = outlen = 0;
        for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
                if (inlen == 0)
                        inblk = crypto_cursor_segment(&cc_in, &inlen);
                if (outlen == 0)
                        outblk = crypto_cursor_segment(&cc_out, &outlen);

                if (inlen < blksz) {
                        crypto_cursor_copydata(&cc_in, blksz, blk);
                        inblk = blk;
                        inlen = blksz;
                }

                if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                        if (outlen < blksz) {
                                outblk = blk;
                                outlen = blksz;
                        }

                        todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
                            blksz);

                        exf->update(ctx, inblk, todo);
                        exf->encrypt_multi(ctx, inblk, outblk, todo);

                        if (outblk == blk) {
                                crypto_cursor_copyback(&cc_out, blksz, blk);
                                outblk = crypto_cursor_segment(&cc_out, &outlen);
                        } else {
                                crypto_cursor_advance(&cc_out, todo);
                                outlen -= todo;
                                outblk += todo;
                        }
                } else {
                        /*
                         * One of the problems with CCM+CBC is that
                         * the authentication is done on the
                         * unencrypted data.  As a result, we have to
                         * decrypt the data twice: once to generate
                         * the tag and a second time after the tag is
                         * verified.
                         */
                        todo = blksz;
                        exf->decrypt(ctx, inblk, blk);
                        exf->update(ctx, blk, todo);
                }

                if (inblk == blk) {
                        inblk = crypto_cursor_segment(&cc_in, &inlen);
                } else {
                        crypto_cursor_advance(&cc_in, todo);
                        inlen -= todo;
                        inblk += todo;
                }
        }
        if (resid > 0) {
                crypto_cursor_copydata(&cc_in, resid, blk);
                if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                        exf->update(ctx, blk, resid);
                        exf->encrypt_last(ctx, blk, blk, resid);
                        crypto_cursor_copyback(&cc_out, resid, blk);
                } else {
                        exf->decrypt_last(ctx, blk, blk, resid);
                        exf->update(ctx, blk, resid);
                }
        }

        /* Finalize MAC */
        exf->final(s.tag, ctx);

        /* Validate tag */
        error = 0;
        if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag2);
                r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen);
                if (r != 0) {
                        error = EBADMSG;
                        goto out;
                }

                /* tag matches, decrypt data */
                exf->reinit(ctx, crp->crp_iv, ivlen);
                crypto_cursor_init(&cc_in, &crp->crp_buf);
                crypto_cursor_advance(&cc_in, crp->crp_payload_start);

                inlen = 0;
                for (resid = crp->crp_payload_length; resid >= blksz;
                     resid -= todo) {
                        if (inlen == 0)
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        if (outlen == 0)
                                outblk = crypto_cursor_segment(&cc_out,
                                    &outlen);

                        if (inlen < blksz) {
                                crypto_cursor_copydata(&cc_in, blksz, blk);
                                inblk = blk;
                                inlen = blksz;
                        }
                        if (outlen < blksz) {
                                outblk = blk;
                                outlen = blksz;
                        }

                        todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
                            blksz);

                        exf->decrypt_multi(ctx, inblk, outblk, todo);

                        if (inblk == blk) {
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        } else {
                                crypto_cursor_advance(&cc_in, todo);
                                inlen -= todo;
                                inblk += todo;
                        }

                        if (outblk == blk) {
                                crypto_cursor_copyback(&cc_out, blksz, blk);
                                outblk = crypto_cursor_segment(&cc_out,
                                    &outlen);
                        } else {
                                crypto_cursor_advance(&cc_out, todo);
                                outlen -= todo;
                                outblk += todo;
                        }
                }
                if (resid > 0) {
                        crypto_cursor_copydata(&cc_in, resid, blk);
                        exf->decrypt_last(ctx, blk, blk, resid);
                        crypto_cursor_copyback(&cc_out, resid, blk);
                }
        } else {
                /* Inject the authentication data */
                crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag);
        }

out:
        explicit_bzero(ctx, exf->ctxsize);
        explicit_bzero(&s, sizeof(s));
        return (error);
}

static int
swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp)
{
        const struct crypto_session_params *csp;
        struct {
                uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))];
                u_char tag[POLY1305_HASH_LEN];
                u_char tag2[POLY1305_HASH_LEN];
        } s;
        u_char *blk = (u_char *)s.blkbuf;
        struct crypto_buffer_cursor cc_in, cc_out;
        const u_char *inblk;
        u_char *outblk;
        size_t inlen, outlen, todo;
        uint64_t *blkp;
        const struct swcr_auth *swa;
        const struct swcr_encdec *swe;
        const struct enc_xform *exf;
        void *ctx;
        int blksz, error, r, resid;

        swa = &ses->swcr_auth;
        swe = &ses->swcr_encdec;
        exf = swe->sw_exf;
        blksz = exf->native_blocksize;
        KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__));

        if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
                return (EINVAL);

        csp = crypto_get_params(crp->crp_session);

        ctx = __builtin_alloca(exf->ctxsize);
        if (crp->crp_cipher_key != NULL)
                exf->setkey(ctx, crp->crp_cipher_key,
                    csp->csp_cipher_klen);
        else
                memcpy(ctx, swe->sw_ctx, exf->ctxsize);
        exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen);

        /* Supply MAC with AAD */
        if (crp->crp_aad != NULL)
                exf->update(ctx, crp->crp_aad, crp->crp_aad_length);
        else
                crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
                    exf->update, ctx);
        if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) {
                /* padding1 */
                memset(blk, 0, POLY1305_BLOCK_LEN);
                exf->update(ctx, blk, POLY1305_BLOCK_LEN -
                    crp->crp_aad_length % POLY1305_BLOCK_LEN);
        }

        /* Do encryption with MAC */
        crypto_cursor_init(&cc_in, &crp->crp_buf);
        crypto_cursor_advance(&cc_in, crp->crp_payload_start);
        if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
                crypto_cursor_init(&cc_out, &crp->crp_obuf);
                crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
        } else
                cc_out = cc_in;

        inlen = outlen = 0;
        if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                for (resid = crp->crp_payload_length; resid >= blksz;
                     resid -= todo) {
                        if (inlen == 0)
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        if (outlen == 0)
                                outblk = crypto_cursor_segment(&cc_out,
                                    &outlen);

                        if (inlen < blksz) {
                                crypto_cursor_copydata(&cc_in, blksz, blk);
                                inblk = blk;
                                inlen = blksz;
                        }

                        if (outlen < blksz) {
                                outblk = blk;
                                outlen = blksz;
                        }

                        todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
                            blksz);

                        exf->encrypt_multi(ctx, inblk, outblk, todo);
                        exf->update(ctx, outblk, todo);

                        if (inblk == blk) {
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        } else {
                                crypto_cursor_advance(&cc_in, todo);
                                inlen -= todo;
                                inblk += todo;
                        }

                        if (outblk == blk) {
                                crypto_cursor_copyback(&cc_out, blksz, blk);
                                outblk = crypto_cursor_segment(&cc_out, &outlen);
                        } else {
                                crypto_cursor_advance(&cc_out, todo);
                                outlen -= todo;
                                outblk += todo;
                        }
                }
                if (resid > 0) {
                        crypto_cursor_copydata(&cc_in, resid, blk);
                        exf->encrypt_last(ctx, blk, blk, resid);
                        crypto_cursor_copyback(&cc_out, resid, blk);
                        exf->update(ctx, blk, resid);
                }
        } else
                crypto_apply(crp, crp->crp_payload_start,
                    crp->crp_payload_length, exf->update, ctx);
        if (crp->crp_payload_length % POLY1305_BLOCK_LEN != 0) {
                /* padding2 */
                memset(blk, 0, POLY1305_BLOCK_LEN);
                exf->update(ctx, blk, POLY1305_BLOCK_LEN -
                    crp->crp_payload_length % POLY1305_BLOCK_LEN);
        }

        /* lengths */
        blkp = (uint64_t *)blk;
        blkp[0] = htole64(crp->crp_aad_length);
        blkp[1] = htole64(crp->crp_payload_length);
        exf->update(ctx, blk, sizeof(uint64_t) * 2);

        /* Finalize MAC */
        exf->final(s.tag, ctx);

        /* Validate tag */
        error = 0;
        if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag2);
                r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen);
                if (r != 0) {
                        error = EBADMSG;
                        goto out;
                }

                /* tag matches, decrypt data */
                crypto_cursor_init(&cc_in, &crp->crp_buf);
                crypto_cursor_advance(&cc_in, crp->crp_payload_start);

                inlen = 0;
                for (resid = crp->crp_payload_length; resid > blksz;
                     resid -= todo) {
                        if (inlen == 0)
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        if (outlen == 0)
                                outblk = crypto_cursor_segment(&cc_out,
                                    &outlen);
                        if (inlen < blksz) {
                                crypto_cursor_copydata(&cc_in, blksz, blk);
                                inblk = blk;
                                inlen = blksz;
                        }
                        if (outlen < blksz) {
                                outblk = blk;
                                outlen = blksz;
                        }

                        todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
                            blksz);

                        exf->decrypt_multi(ctx, inblk, outblk, todo);

                        if (inblk == blk) {
                                inblk = crypto_cursor_segment(&cc_in, &inlen);
                        } else {
                                crypto_cursor_advance(&cc_in, todo);
                                inlen -= todo;
                                inblk += todo;
                        }

                        if (outblk == blk) {
                                crypto_cursor_copyback(&cc_out, blksz, blk);
                                outblk = crypto_cursor_segment(&cc_out,
                                    &outlen);
                        } else {
                                crypto_cursor_advance(&cc_out, todo);
                                outlen -= todo;
                                outblk += todo;
                        }
                }
                if (resid > 0) {
                        crypto_cursor_copydata(&cc_in, resid, blk);
                        exf->decrypt_last(ctx, blk, blk, resid);
                        crypto_cursor_copyback(&cc_out, resid, blk);
                }
        } else {
                /* Inject the authentication data */
                crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
                    s.tag);
        }

out:
        explicit_bzero(ctx, exf->ctxsize);
        explicit_bzero(&s, sizeof(s));
        return (error);
}

/*
 * Apply a cipher and a digest to perform EtA.
 */
static int
swcr_eta(const struct swcr_session *ses, struct cryptop *crp)
{
        int error;

        if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
                error = swcr_encdec(ses, crp);
                if (error == 0)
                        error = swcr_authcompute(ses, crp);
        } else {
                error = swcr_authcompute(ses, crp);
                if (error == 0)
                        error = swcr_encdec(ses, crp);
        }
        return (error);
}

/*
 * Apply a compression/decompression algorithm
 */
static int
swcr_compdec(const struct swcr_session *ses, struct cryptop *crp)
{
        const struct comp_algo *cxf;
        uint8_t *data, *out;
        int adj;
        uint32_t result;

        cxf = ses->swcr_compdec.sw_cxf;

        /* We must handle the whole buffer of data in one time
         * then if there is not all the data in the mbuf, we must
         * copy in a buffer.
         */

        data = malloc(crp->crp_payload_length, M_CRYPTO_DATA,  M_NOWAIT);
        if (data == NULL)
                return (EINVAL);
        crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length,
            data);

        if (CRYPTO_OP_IS_COMPRESS(crp->crp_op))
                result = cxf->compress(data, crp->crp_payload_length, &out);
        else
                result = cxf->decompress(data, crp->crp_payload_length, &out);

        free(data, M_CRYPTO_DATA);
        if (result == 0)
                return (EINVAL);
        crp->crp_olen = result;

        /* Check the compressed size when doing compression */
        if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) {
                if (result >= crp->crp_payload_length) {
                        /* Compression was useless, we lost time */
                        free(out, M_CRYPTO_DATA);
                        return (0);
                }
        }

        /* Copy back the (de)compressed data. m_copyback is
         * extending the mbuf as necessary.
         */
        crypto_copyback(crp, crp->crp_payload_start, result, out);
        if (result < crp->crp_payload_length) {
                switch (crp->crp_buf.cb_type) {
                case CRYPTO_BUF_MBUF:
                case CRYPTO_BUF_SINGLE_MBUF:
                        adj = result - crp->crp_payload_length;
                        m_adj(crp->crp_buf.cb_mbuf, adj);
                        break;
                case CRYPTO_BUF_UIO: {
                        struct uio *uio = crp->crp_buf.cb_uio;
                        int ind;

                        adj = crp->crp_payload_length - result;
                        ind = uio->uio_iovcnt - 1;

                        while (adj > 0 && ind >= 0) {
                                if (adj < uio->uio_iov[ind].iov_len) {
                                        uio->uio_iov[ind].iov_len -= adj;
                                        break;
                                }

                                adj -= uio->uio_iov[ind].iov_len;
                                uio->uio_iov[ind].iov_len = 0;
                                ind--;
                                uio->uio_iovcnt--;
                        }
                        }
                        break;
                case CRYPTO_BUF_VMPAGE:
                        adj = crp->crp_payload_length - result;
                        crp->crp_buf.cb_vm_page_len -= adj;
                        break;
                default:
                        break;
                }
        }
        free(out, M_CRYPTO_DATA);
        return 0;
}

static int
swcr_setup_cipher(struct swcr_session *ses,
    const struct crypto_session_params *csp)
{
        struct swcr_encdec *swe;
        const struct enc_xform *txf;
        int error;

        swe = &ses->swcr_encdec;
        txf = crypto_cipher(csp);
        if (csp->csp_cipher_key != NULL) {
                if (txf->ctxsize != 0) {
                        swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA,
                            M_NOWAIT);
                        if (swe->sw_ctx == NULL)
                                return (ENOMEM);
                }
                error = txf->setkey(swe->sw_ctx,
                    csp->csp_cipher_key, csp->csp_cipher_klen);
                if (error)
                        return (error);
        }
        swe->sw_exf = txf;
        return (0);
}

static int
swcr_setup_auth(struct swcr_session *ses,
    const struct crypto_session_params *csp)
{
        struct swcr_auth *swa;
        const struct auth_hash *axf;

        swa = &ses->swcr_auth;

        axf = crypto_auth_hash(csp);
        swa->sw_axf = axf;
        if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
                return (EINVAL);
        if (csp->csp_auth_mlen == 0)
                swa->sw_mlen = axf->hashsize;
        else
                swa->sw_mlen = csp->csp_auth_mlen;
        if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) {
                swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
                    M_NOWAIT);
                if (swa->sw_ictx == NULL)
                        return (ENOBUFS);
        }

        switch (csp->csp_auth_alg) {
        case CRYPTO_SHA1_HMAC:
        case CRYPTO_SHA2_224_HMAC:
        case CRYPTO_SHA2_256_HMAC:
        case CRYPTO_SHA2_384_HMAC:
        case CRYPTO_SHA2_512_HMAC:
        case CRYPTO_RIPEMD160_HMAC:
                swa->sw_hmac = true;
                if (csp->csp_auth_key != NULL) {
                        swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
                            M_NOWAIT);
                        if (swa->sw_octx == NULL)
                                return (ENOBUFS);
                        hmac_init_ipad(axf, csp->csp_auth_key,
                            csp->csp_auth_klen, swa->sw_ictx);
                        hmac_init_opad(axf, csp->csp_auth_key,
                            csp->csp_auth_klen, swa->sw_octx);
                }
                break;
        case CRYPTO_RIPEMD160:
        case CRYPTO_SHA1:
        case CRYPTO_SHA2_224:
        case CRYPTO_SHA2_256:
        case CRYPTO_SHA2_384:
        case CRYPTO_SHA2_512:
        case CRYPTO_NULL_HMAC:
                axf->Init(swa->sw_ictx);
                break;
        case CRYPTO_AES_NIST_GMAC:
        case CRYPTO_AES_CCM_CBC_MAC:
        case CRYPTO_POLY1305:
                if (csp->csp_auth_key != NULL) {
                        axf->Init(swa->sw_ictx);
                        axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
                            csp->csp_auth_klen);
                }
                break;
        case CRYPTO_BLAKE2B:
        case CRYPTO_BLAKE2S:
                /*
                 * Blake2b and Blake2s support an optional key but do
                 * not require one.
                 */
                if (csp->csp_auth_klen == 0)
                        axf->Init(swa->sw_ictx);
                else if (csp->csp_auth_key != NULL)
                        axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
                            csp->csp_auth_klen);
                break;
        }

        if (csp->csp_mode == CSP_MODE_DIGEST) {
                switch (csp->csp_auth_alg) {
                case CRYPTO_AES_NIST_GMAC:
                        ses->swcr_process = swcr_gmac;
                        break;
                case CRYPTO_AES_CCM_CBC_MAC:
                        ses->swcr_process = swcr_ccm_cbc_mac;
                        break;
                default:
                        ses->swcr_process = swcr_authcompute;
                }
        }

        return (0);
}

static int
swcr_setup_aead(struct swcr_session *ses,
    const struct crypto_session_params *csp)
{
        struct swcr_auth *swa;
        int error;

        error = swcr_setup_cipher(ses, csp);
        if (error)
                return (error);

        swa = &ses->swcr_auth;
        if (csp->csp_auth_mlen == 0)
                swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize;
        else
                swa->sw_mlen = csp->csp_auth_mlen;
        return (0);
}

static bool
swcr_auth_supported(const struct crypto_session_params *csp)
{
        const struct auth_hash *axf;

        axf = crypto_auth_hash(csp);
        if (axf == NULL)
                return (false);
        switch (csp->csp_auth_alg) {
        case CRYPTO_SHA1_HMAC:
        case CRYPTO_SHA2_224_HMAC:
        case CRYPTO_SHA2_256_HMAC:
        case CRYPTO_SHA2_384_HMAC:
        case CRYPTO_SHA2_512_HMAC:
        case CRYPTO_NULL_HMAC:
        case CRYPTO_RIPEMD160_HMAC:
                break;
        case CRYPTO_AES_NIST_GMAC:
                switch (csp->csp_auth_klen * 8) {
                case 128:
                case 192:
                case 256:
                        break;
                default:
                        return (false);
                }
                if (csp->csp_auth_key == NULL)
                        return (false);
                if (csp->csp_ivlen != AES_GCM_IV_LEN)
                        return (false);
                break;
        case CRYPTO_POLY1305:
                if (csp->csp_auth_klen != POLY1305_KEY_LEN)
                        return (false);
                break;
        case CRYPTO_AES_CCM_CBC_MAC:
                switch (csp->csp_auth_klen * 8) {
                case 128:
                case 192:
                case 256:
                        break;
                default:
                        return (false);
                }
                if (csp->csp_auth_key == NULL)
                        return (false);
                break;
        }
        return (true);
}

static bool
swcr_cipher_supported(const struct crypto_session_params *csp)
{
        const struct enc_xform *txf;

        txf = crypto_cipher(csp);
        if (txf == NULL)
                return (false);
        if (csp->csp_cipher_alg != CRYPTO_NULL_CBC &&
            txf->ivsize != csp->csp_ivlen)
                return (false);
        return (true);
}

#define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)

static int
swcr_probesession(device_t dev, const struct crypto_session_params *csp)
{
        if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
                return (EINVAL);
        switch (csp->csp_mode) {
        case CSP_MODE_COMPRESS:
                switch (csp->csp_cipher_alg) {
                case CRYPTO_DEFLATE_COMP:
                        break;
                default:
                        return (EINVAL);
                }
                break;
        case CSP_MODE_CIPHER:
                switch (csp->csp_cipher_alg) {
                case CRYPTO_AES_NIST_GCM_16:
                case CRYPTO_AES_CCM_16:
                case CRYPTO_CHACHA20_POLY1305:
                case CRYPTO_XCHACHA20_POLY1305:
                        return (EINVAL);
                default:
                        if (!swcr_cipher_supported(csp))
                                return (EINVAL);
                        break;
                }
                break;
        case CSP_MODE_DIGEST:
                if (!swcr_auth_supported(csp))
                        return (EINVAL);
                break;
        case CSP_MODE_AEAD:
                switch (csp->csp_cipher_alg) {
                case CRYPTO_AES_NIST_GCM_16:
                case CRYPTO_AES_CCM_16:
                        switch (csp->csp_cipher_klen * 8) {
                        case 128:
                        case 192:
                        case 256:
                                break;
                        default:
                                return (EINVAL);
                        }
                        break;
                case CRYPTO_CHACHA20_POLY1305:
                case CRYPTO_XCHACHA20_POLY1305:
                        break;
                default:
                        return (EINVAL);
                }
                break;
        case CSP_MODE_ETA:
                /* AEAD algorithms cannot be used for EtA. */
                switch (csp->csp_cipher_alg) {
                case CRYPTO_AES_NIST_GCM_16:
                case CRYPTO_AES_CCM_16:
                case CRYPTO_CHACHA20_POLY1305:
                case CRYPTO_XCHACHA20_POLY1305:
                        return (EINVAL);
                }
                switch (csp->csp_auth_alg) {
                case CRYPTO_AES_NIST_GMAC:
                case CRYPTO_AES_CCM_CBC_MAC:
                        return (EINVAL);
                }

                if (!swcr_cipher_supported(csp) ||
                    !swcr_auth_supported(csp))
                        return (EINVAL);
                break;
        default:
                return (EINVAL);
        }

        return (CRYPTODEV_PROBE_SOFTWARE);
}

/*
 * Generate a new software session.
 */
static int
swcr_newsession(device_t dev, crypto_session_t cses,
    const struct crypto_session_params *csp)
{
        struct swcr_session *ses;
        const struct comp_algo *cxf;
        int error;

        ses = crypto_get_driver_session(cses);

        error = 0;
        switch (csp->csp_mode) {
        case CSP_MODE_COMPRESS:
                switch (csp->csp_cipher_alg) {
                case CRYPTO_DEFLATE_COMP:
                        cxf = &comp_algo_deflate;
                        break;
#ifdef INVARIANTS
                default:
                        panic("bad compression algo");
#endif
                }
                ses->swcr_compdec.sw_cxf = cxf;
                ses->swcr_process = swcr_compdec;
                break;
        case CSP_MODE_CIPHER:
                switch (csp->csp_cipher_alg) {
                case CRYPTO_NULL_CBC:
                        ses->swcr_process = swcr_null;
                        break;
#ifdef INVARIANTS
                case CRYPTO_AES_NIST_GCM_16:
                case CRYPTO_AES_CCM_16:
                case CRYPTO_CHACHA20_POLY1305:
                case CRYPTO_XCHACHA20_POLY1305:
                        panic("bad cipher algo");
#endif
                default:
                        error = swcr_setup_cipher(ses, csp);
                        if (error == 0)
                                ses->swcr_process = swcr_encdec;
                }
                break;
        case CSP_MODE_DIGEST:
                error = swcr_setup_auth(ses, csp);
                break;
        case CSP_MODE_AEAD:
                switch (csp->csp_cipher_alg) {
                case CRYPTO_AES_NIST_GCM_16:
                        error = swcr_setup_aead(ses, csp);
                        if (error == 0)
                                ses->swcr_process = swcr_gcm;
                        break;
                case CRYPTO_AES_CCM_16:
                        error = swcr_setup_aead(ses, csp);
                        if (error == 0)
                                ses->swcr_process = swcr_ccm;
                        break;
                case CRYPTO_CHACHA20_POLY1305:
                case CRYPTO_XCHACHA20_POLY1305:
                        error = swcr_setup_aead(ses, csp);
                        if (error == 0)
                                ses->swcr_process = swcr_chacha20_poly1305;
                        break;
#ifdef INVARIANTS
                default:
                        panic("bad aead algo");
#endif
                }
                break;
        case CSP_MODE_ETA:
#ifdef INVARIANTS
                switch (csp->csp_cipher_alg) {
                case CRYPTO_AES_NIST_GCM_16:
                case CRYPTO_AES_CCM_16:
                case CRYPTO_CHACHA20_POLY1305:
                case CRYPTO_XCHACHA20_POLY1305:
                        panic("bad eta cipher algo");
                }
                switch (csp->csp_auth_alg) {
                case CRYPTO_AES_NIST_GMAC:
                case CRYPTO_AES_CCM_CBC_MAC:
                        panic("bad eta auth algo");
                }
#endif

                error = swcr_setup_auth(ses, csp);
                if (error)
                        break;
                if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) {
                        /* Effectively degrade to digest mode. */
                        ses->swcr_process = swcr_authcompute;
                        break;
                }

                error = swcr_setup_cipher(ses, csp);
                if (error == 0)
                        ses->swcr_process = swcr_eta;
                break;
        default:
                error = EINVAL;
        }

        if (error)
                swcr_freesession(dev, cses);
        return (error);
}

static void
swcr_freesession(device_t dev, crypto_session_t cses)
{
        struct swcr_session *ses;

        ses = crypto_get_driver_session(cses);

        zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA);
        zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA);
        zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA);
}

/*
 * Process a software request.
 */
static int
swcr_process(device_t dev, struct cryptop *crp, int hint)
{
        struct swcr_session *ses;

        ses = crypto_get_driver_session(crp->crp_session);

        crp->crp_etype = ses->swcr_process(ses, crp);

        crypto_done(crp);
        return (0);
}

static void
swcr_identify(driver_t *drv, device_t parent)
{
        /* NB: order 10 is so we get attached after h/w devices */
        if (device_find_child(parent, "cryptosoft", DEVICE_UNIT_ANY) == NULL &&
            BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0)
                panic("cryptosoft: could not attach");
}

static int
swcr_probe(device_t dev)
{
        device_set_desc(dev, "software crypto");
        device_quiet(dev);
        return (BUS_PROBE_NOWILDCARD);
}

static int
swcr_attach(device_t dev)
{

        swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session),
                        CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
        if (swcr_id < 0) {
                device_printf(dev, "cannot initialize!");
                return (ENXIO);
        }

        return (0);
}

static int
swcr_detach(device_t dev)
{
        crypto_unregister_all(swcr_id);
        return 0;
}

static device_method_t swcr_methods[] = {
        DEVMETHOD(device_identify,      swcr_identify),
        DEVMETHOD(device_probe,         swcr_probe),
        DEVMETHOD(device_attach,        swcr_attach),
        DEVMETHOD(device_detach,        swcr_detach),

        DEVMETHOD(cryptodev_probesession, swcr_probesession),
        DEVMETHOD(cryptodev_newsession, swcr_newsession),
        DEVMETHOD(cryptodev_freesession,swcr_freesession),
        DEVMETHOD(cryptodev_process,    swcr_process),

        {0, 0},
};

static driver_t swcr_driver = {
        "cryptosoft",
        swcr_methods,
        0,              /* NB: no softc */
};

/*
 * NB: We explicitly reference the crypto module so we
 * get the necessary ordering when built as a loadable
 * module.  This is required because we bundle the crypto
 * module code together with the cryptosoft driver (otherwise
 * normal module dependencies would handle things).
 */
extern int crypto_modevent(struct module *, int, void *);
/* XXX where to attach */
DRIVER_MODULE(cryptosoft, nexus, swcr_driver, crypto_modevent, NULL);
MODULE_VERSION(cryptosoft, 1);
MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);