sys/crypto/aes/aes_bear64.c

root/sys/crypto/aes/aes_bear64.c
/*      $NetBSD: aes_bear64.c,v 1.1 2025/11/23 22:44:13 riastradh Exp $ */

/*-
 * Copyright (c) 2025 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(1, "$NetBSD: aes_bear64.c,v 1.1 2025/11/23 22:44:13 riastradh Exp $");

#include <sys/types.h>
#include <sys/endian.h>

#ifdef _KERNEL
#include <sys/systm.h>
#else
#include <assert.h>
#include <err.h>
#include <string.h>
#define KASSERT                 assert
#define panic(fmt, args...)     err(1, fmt, args)
#endif

#include <crypto/aes/aes.h>
#include <crypto/aes/aes_bear64.h>
#include <crypto/aes/aes_impl.h>

static void
aesbear64_setkey(uint64_t rk[static 30], const void *key, uint32_t nrounds)
{
        size_t key_len;

        switch (nrounds) {
        case 10:
                key_len = 16;
                break;
        case 12:
                key_len = 24;
                break;
        case 14:
                key_len = 32;
                break;
        default:
                panic("invalid AES nrounds: %u", nrounds);
        }

        br_aes_ct64_keysched(rk, key, key_len);
}

static void
aesbear64_setenckey(struct aesenc *enc, const uint8_t *key, uint32_t nrounds)
{

        aesbear64_setkey(enc->aese_aes.aes_rk64, key, nrounds);
}

static void
aesbear64_setdeckey(struct aesdec *dec, const uint8_t *key, uint32_t nrounds)
{

        /*
         * BearSSL computes InvMixColumns on the fly -- no need for
         * distinct decryption round keys.
         */
        aesbear64_setkey(dec->aesd_aes.aes_rk64, key, nrounds);
}

static void
aesbear64_enc(const struct aesenc *enc, const uint8_t in[static 16],
    uint8_t out[static 16], uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);

        /* Load input block interleaved with garbage blocks.  */
        w[0] = le32dec(in + 4*0);
        w[1] = le32dec(in + 4*1);
        w[2] = le32dec(in + 4*2);
        w[3] = le32dec(in + 4*3);
        br_aes_ct64_interleave_in(&q[0], &q[4], w);
        q[1] = q[2] = q[3] = 0;
        q[5] = q[6] = q[7] = 0;

        /* Transform to bitslice, encrypt, transform from bitslice.  */
        br_aes_ct64_ortho(q);
        br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
        br_aes_ct64_ortho(q);

        /* Store output block.  */
        br_aes_ct64_interleave_out(w, q[0], q[4]);
        le32enc(out + 4*0, w[0]);
        le32enc(out + 4*1, w[1]);
        le32enc(out + 4*2, w[2]);
        le32enc(out + 4*3, w[3]);

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static void
aesbear64_dec(const struct aesdec *dec, const uint8_t in[static 16],
    uint8_t out[static 16], uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, dec->aesd_aes.aes_rk64);

        /* Load input block interleaved with garbage blocks.  */
        w[0] = le32dec(in + 4*0);
        w[1] = le32dec(in + 4*1);
        w[2] = le32dec(in + 4*2);
        w[3] = le32dec(in + 4*3);
        br_aes_ct64_interleave_in(&q[0], &q[4], w);
        q[1] = q[2] = q[3] = 0;
        q[5] = q[6] = q[7] = 0;

        /* Transform to bitslice, decrypt, transform from bitslice.  */
        br_aes_ct64_ortho(q);
        br_aes_ct64_bitslice_decrypt(nrounds, sk_exp, q);
        br_aes_ct64_ortho(q);

        /* Store output block.  */
        br_aes_ct64_interleave_out(w, q[0], q[4]);
        le32enc(out + 4*0, w[0]);
        le32enc(out + 4*1, w[1]);
        le32enc(out + 4*2, w[2]);
        le32enc(out + 4*3, w[3]);

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static void
aesbear64_cbc_enc(const struct aesenc *enc, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
    uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];
        uint32_t cv0, cv1, cv2, cv3;

        KASSERT(nbytes % 16 == 0);

        /* Skip if there's nothing to do.  */
        if (nbytes == 0)
                return;

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);

        /* Initialize garbage blocks.  */
        q[1] = q[2] = q[3] = 0;
        q[5] = q[6] = q[7] = 0;

        /* Load IV.  */
        cv0 = le32dec(iv + 4*0);
        cv1 = le32dec(iv + 4*1);
        cv2 = le32dec(iv + 4*2);
        cv3 = le32dec(iv + 4*3);

        for (; nbytes; nbytes -= 16, in += 16, out += 16) {
                /* Load input block and apply CV.  */
                w[0] = cv0 ^ le32dec(in + 4*0);
                w[1] = cv1 ^ le32dec(in + 4*1);
                w[2] = cv2 ^ le32dec(in + 4*2);
                w[3] = cv3 ^ le32dec(in + 4*3);
                br_aes_ct64_interleave_in(&q[0], &q[4], w);

                /* Transform to bitslice, encrypt, transform from bitslice.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                /* Remember ciphertext as CV and store output block.  */
                br_aes_ct64_interleave_out(w, q[0], q[4]);
                cv0 = w[0];
                cv1 = w[1];
                cv2 = w[2];
                cv3 = w[3];
                le32enc(out + 4*0, cv0);
                le32enc(out + 4*1, cv1);
                le32enc(out + 4*2, cv2);
                le32enc(out + 4*3, cv3);
        }

        /* Store updated IV.  */
        le32enc(iv + 4*0, cv0);
        le32enc(iv + 4*1, cv1);
        le32enc(iv + 4*2, cv2);
        le32enc(iv + 4*3, cv3);

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static void
aesbear64_cbc_dec(const struct aesdec *dec, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
    uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];
        uint32_t cv0, cv1, cv2, cv3, iv0, iv1, iv2, iv3;
        unsigned i;

        KASSERT(nbytes % 16 == 0);

        /* Skip if there's nothing to do.  */
        if (nbytes == 0)
                return;

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, dec->aesd_aes.aes_rk64);

        /* Load the IV.  */
        iv0 = le32dec(iv + 4*0);
        iv1 = le32dec(iv + 4*1);
        iv2 = le32dec(iv + 4*2);
        iv3 = le32dec(iv + 4*3);

        /* Load the last cipher block.  */
        cv0 = le32dec(in + nbytes - 16 + 4*0);
        cv1 = le32dec(in + nbytes - 16 + 4*1);
        cv2 = le32dec(in + nbytes - 16 + 4*2);
        cv3 = le32dec(in + nbytes - 16 + 4*3);

        /* Store the updated IV.  */
        le32enc(iv + 4*0, cv0);
        le32enc(iv + 4*1, cv1);
        le32enc(iv + 4*2, cv2);
        le32enc(iv + 4*3, cv3);

        /* Handle the last cipher block separately if odd number.  */
        if (nbytes % 64) {
                unsigned n = (nbytes % 64)/16;

                KASSERT(n == 1 || n == 2 || n == 3);

                for (i = 4; i --> n;)
                        q[i] = q[4 + i] = 0;
                KASSERT(i == n - 1);
                w[0] = cv0;     /* le32dec(in + nbytes - 16*n + 16*i + 4*0) */
                w[1] = cv1;     /* le32dec(in + nbytes - 16*n + 16*i + 4*1) */
                w[2] = cv2;     /* le32dec(in + nbytes - 16*n + 16*i + 4*2) */
                w[3] = cv3;     /* le32dec(in + nbytes - 16*n + 16*i + 4*3) */
                br_aes_ct64_interleave_in(&q[i], &q[4 + i], w);
                while (i --> 0) {
                        w[0] = le32dec(in + nbytes - 16*n + 16*i + 4*0);
                        w[1] = le32dec(in + nbytes - 16*n + 16*i + 4*1);
                        w[2] = le32dec(in + nbytes - 16*n + 16*i + 4*2);
                        w[3] = le32dec(in + nbytes - 16*n + 16*i + 4*3);
                        br_aes_ct64_interleave_in(&q[i], &q[4 + i], w);
                }

                /* Decrypt.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_decrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                for (i = n; i --> 1;) {
                        br_aes_ct64_interleave_out(w, q[i], q[4 + i]);
                        cv0 = le32dec(in + nbytes - 16*n + 16*(i - 1) + 4*0);
                        cv1 = le32dec(in + nbytes - 16*n + 16*(i - 1) + 4*1);
                        cv2 = le32dec(in + nbytes - 16*n + 16*(i - 1) + 4*2);
                        cv3 = le32dec(in + nbytes - 16*n + 16*(i - 1) + 4*3);
                        le32enc(out + nbytes - 16*n + 16*i + 4*0, w[0] ^ cv0);
                        le32enc(out + nbytes - 16*n + 16*i + 4*1, w[1] ^ cv1);
                        le32enc(out + nbytes - 16*n + 16*i + 4*2, w[2] ^ cv2);
                        le32enc(out + nbytes - 16*n + 16*i + 4*3, w[3] ^ cv3);
                }
                br_aes_ct64_interleave_out(w, q[0], q[4]);

                /* If this was the only cipher block, we're done.  */
                nbytes -= nbytes % 64;
                if (nbytes == 0)
                        goto out;

                /*
                 * Otherwise, load up the previous cipher block, and
                 * store the output block.
                 */
                cv0 = le32dec(in + nbytes - 16 + 4*0);
                cv1 = le32dec(in + nbytes - 16 + 4*1);
                cv2 = le32dec(in + nbytes - 16 + 4*2);
                cv3 = le32dec(in + nbytes - 16 + 4*3);
                le32enc(out + nbytes + 4*0, cv0 ^ w[0]);
                le32enc(out + nbytes + 4*1, cv1 ^ w[1]);
                le32enc(out + nbytes + 4*2, cv2 ^ w[2]);
                le32enc(out + nbytes + 4*3, cv3 ^ w[3]);
        }

        for (;;) {
                KASSERT(nbytes >= 64);

                /* Load the input blocks.  */
                w[0] = cv0;     /* le32dec(in + nbytes - 64 + 16*i + 4*0) */
                w[1] = cv1;     /* le32dec(in + nbytes - 64 + 16*i + 4*1) */
                w[2] = cv2;     /* le32dec(in + nbytes - 64 + 16*i + 4*2) */
                w[3] = cv3;     /* le32dec(in + nbytes - 64 + 16*i + 4*3) */
                br_aes_ct64_interleave_in(&q[3], &q[7], w);
                for (i = 3; i --> 0;) {
                        w[0] = le32dec(in + nbytes - 64 + 16*i + 4*0);
                        w[1] = le32dec(in + nbytes - 64 + 16*i + 4*1);
                        w[2] = le32dec(in + nbytes - 64 + 16*i + 4*2);
                        w[3] = le32dec(in + nbytes - 64 + 16*i + 4*3);
                        br_aes_ct64_interleave_in(&q[i], &q[4 + i], w);
                }

                /* Decrypt.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_decrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                /* Store the upper output blocks.  */
                for (i = 4; i --> 1;) {
                        br_aes_ct64_interleave_out(w, q[i], q[4 + i]);
                        cv0 = le32dec(in + nbytes - 64 + 16*(i - 1) + 4*0);
                        cv1 = le32dec(in + nbytes - 64 + 16*(i - 1) + 4*1);
                        cv2 = le32dec(in + nbytes - 64 + 16*(i - 1) + 4*2);
                        cv3 = le32dec(in + nbytes - 64 + 16*(i - 1) + 4*3);
                        le32enc(out + nbytes - 64 + 16*i + 4*0, w[0] ^ cv0);
                        le32enc(out + nbytes - 64 + 16*i + 4*1, w[1] ^ cv1);
                        le32enc(out + nbytes - 64 + 16*i + 4*2, w[2] ^ cv2);
                        le32enc(out + nbytes - 64 + 16*i + 4*3, w[3] ^ cv3);
                }

                /* Prepare the first output block.  */
                br_aes_ct64_interleave_out(w, q[0], q[4]);

                /* Stop if we've reached the first output block.  */
                nbytes -= 64;
                if (nbytes == 0)
                        goto out;

                /*
                 * Load the preceding cipher block, and apply it as the
                 * chaining value to this one.
                 */
                cv0 = le32dec(in + nbytes - 16 + 4*0);
                cv1 = le32dec(in + nbytes - 16 + 4*1);
                cv2 = le32dec(in + nbytes - 16 + 4*2);
                cv3 = le32dec(in + nbytes - 16 + 4*3);
                le32enc(out + nbytes + 4*0, w[0] ^ cv0);
                le32enc(out + nbytes + 4*1, w[1] ^ cv1);
                le32enc(out + nbytes + 4*2, w[2] ^ cv2);
                le32enc(out + nbytes + 4*3, w[3] ^ cv3);
        }

out:    /* Store the first output block.  */
        le32enc(out + 4*0, w[0] ^ iv0);
        le32enc(out + 4*1, w[1] ^ iv1);
        le32enc(out + 4*2, w[2] ^ iv2);
        le32enc(out + 4*3, w[3] ^ iv3);

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static inline void
aesbear64_xts_update(uint32_t *t0, uint32_t *t1, uint32_t *t2, uint32_t *t3)
{
        uint32_t s0, s1, s2, s3;

        s0 = *t0 >> 31;
        s1 = *t1 >> 31;
        s2 = *t2 >> 31;
        s3 = *t3 >> 31;
        *t0 = (*t0 << 1) ^ (-s3 & 0x87);
        *t1 = (*t1 << 1) ^ s0;
        *t2 = (*t2 << 1) ^ s1;
        *t3 = (*t3 << 1) ^ s2;
}

static int
aesbear64_xts_update_selftest(void)
{
        static const struct {
                uint32_t in[4], out[4];
        } cases[] = {
                { {1}, {2} },
                { {0x80000000U,0,0,0}, {0,1,0,0} },
                { {0,0x80000000U,0,0}, {0,0,1,0} },
                { {0,0,0x80000000U,0}, {0,0,0,1} },
                { {0,0,0,0x80000000U}, {0x87,0,0,0} },
                { {0,0x80000000U,0,0x80000000U}, {0x87,0,1,0} },
        };
        unsigned i;
        uint32_t t0, t1, t2, t3;

        for (i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) {
                t0 = cases[i].in[0];
                t1 = cases[i].in[1];
                t2 = cases[i].in[2];
                t3 = cases[i].in[3];
                aesbear64_xts_update(&t0, &t1, &t2, &t3);
                if (t0 != cases[i].out[0] ||
                    t1 != cases[i].out[1] ||
                    t2 != cases[i].out[2] ||
                    t3 != cases[i].out[3])
                        return -1;
        }

        /* Success!  */
        return 0;
}

static void
aesbear64_xts_enc(const struct aesenc *enc, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16],
    uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];
        uint32_t t0, t1, t2, t3, u0, u1, u2, u3;
        unsigned i;

        KASSERT(nbytes % 16 == 0);

        /* Skip if there's nothing to do.  */
        if (nbytes == 0)
                return;

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);

        /* Load tweak.  */
        t0 = le32dec(tweak + 4*0);
        t1 = le32dec(tweak + 4*1);
        t2 = le32dec(tweak + 4*2);
        t3 = le32dec(tweak + 4*3);

        /* Handle the first blocks separately if odd number.  */
        if (nbytes % 64) {
                unsigned n = (nbytes % 64)/16;

                /* Load up the first blocks and garbage.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < n; i++) {
                        w[0] = le32dec(in + 16*i + 4*0) ^ u0;
                        w[1] = le32dec(in + 16*i + 4*1) ^ u1;
                        w[2] = le32dec(in + 16*i + 4*2) ^ u2;
                        w[3] = le32dec(in + 16*i + 4*3) ^ u3;
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                        br_aes_ct64_interleave_in(&q[i], &q[4 + i], w);
                }
                for (; i < 4; i++)
                        q[i] = q[4 + i] = 0;

                /* Encrypt up to three blocks.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                /* Store up to three blocks.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < n; i++) {
                        br_aes_ct64_interleave_out(w, q[i], q[4 + i]);
                        le32enc(out + 16*i + 4*0, w[0] ^ u0);
                        le32enc(out + 16*i + 4*1, w[1] ^ u1);
                        le32enc(out + 16*i + 4*2, w[2] ^ u2);
                        le32enc(out + 16*i + 4*3, w[3] ^ u3);
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                }

                /* Advance to the next block.  */
                t0 = u0, t1 = u1, t2 = u2, t3 = u3;
                if ((nbytes -= 16*n) == 0)
                        goto out;
                in += 16*n;
                out += 16*n;
        }

        do {
                KASSERT(nbytes >= 64);

                /* Load four blocks.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < 4; i++) {
                        w[0] = le32dec(in + 16*i + 4*0) ^ u0;
                        w[1] = le32dec(in + 16*i + 4*1) ^ u1;
                        w[2] = le32dec(in + 16*i + 4*2) ^ u2;
                        w[3] = le32dec(in + 16*i + 4*3) ^ u3;
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                        br_aes_ct64_interleave_in(&q[i], &q[4 + i], w);
                }

                /* Encrypt four blocks.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                /* Store four blocks.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < 4; i++) {
                        br_aes_ct64_interleave_out(w, q[i], q[4 + i]);
                        le32enc(out + 16*i + 4*0, w[0] ^ u0);
                        le32enc(out + 16*i + 4*1, w[1] ^ u1);
                        le32enc(out + 16*i + 4*2, w[2] ^ u2);
                        le32enc(out + 16*i + 4*3, w[3] ^ u3);
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                }

                /* Advance to the next pair of blocks.  */
                t0 = u0, t1 = u1, t2 = u2, t3 = u3;
                in += 64;
                out += 64;
        } while (nbytes -= 64, nbytes);

out:    /* Store the updated tweak.  */
        le32enc(tweak + 4*0, t0);
        le32enc(tweak + 4*1, t1);
        le32enc(tweak + 4*2, t2);
        le32enc(tweak + 4*3, t3);

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static void
aesbear64_xts_dec(const struct aesdec *dec, const uint8_t in[static 16],
    uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16],
    uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];
        uint32_t t0, t1, t2, t3, u0, u1, u2, u3;
        unsigned i;

        KASSERT(nbytes % 16 == 0);

        /* Skip if there's nothing to do.  */
        if (nbytes == 0)
                return;

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, dec->aesd_aes.aes_rk64);

        /* Load tweak.  */
        t0 = le32dec(tweak + 4*0);
        t1 = le32dec(tweak + 4*1);
        t2 = le32dec(tweak + 4*2);
        t3 = le32dec(tweak + 4*3);

        /* Handle the first blocks separately if odd number.  */
        if (nbytes % 64) {
                unsigned n = (nbytes % 64)/16;

                /* Load up the first blocks and garbage.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < n; i++) {
                        w[0] = le32dec(in + 16*i + 4*0) ^ u0;
                        w[1] = le32dec(in + 16*i + 4*1) ^ u1;
                        w[2] = le32dec(in + 16*i + 4*2) ^ u2;
                        w[3] = le32dec(in + 16*i + 4*3) ^ u3;
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                        br_aes_ct64_interleave_in(&q[i], &q[4 + i], w);
                }
                for (; i < 4; i++)
                        q[i] = q[4 + i] = 0;

                /* Decrypt up to three blocks.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_decrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                /* Store up to three blocks.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < n; i++) {
                        br_aes_ct64_interleave_out(w, q[i], q[4 + i]);
                        le32enc(out + 16*i + 4*0, w[0] ^ u0);
                        le32enc(out + 16*i + 4*1, w[1] ^ u1);
                        le32enc(out + 16*i + 4*2, w[2] ^ u2);
                        le32enc(out + 16*i + 4*3, w[3] ^ u3);
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                }

                /* Advance to the next block.  */
                t0 = u0, t1 = u1, t2 = u2, t3 = u3;
                if ((nbytes -= 16*n) == 0)
                        goto out;
                in += 16*n;
                out += 16*n;
        }

        do {
                KASSERT(nbytes >= 64);

                /* Load four blocks.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < 4; i++) {
                        w[0] = le32dec(in + 16*i + 4*0) ^ u0;
                        w[1] = le32dec(in + 16*i + 4*1) ^ u1;
                        w[2] = le32dec(in + 16*i + 4*2) ^ u2;
                        w[3] = le32dec(in + 16*i + 4*3) ^ u3;
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                        br_aes_ct64_interleave_in(&q[i], &q[4 + i], w);
                }

                /* Decrypt four blocks.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_decrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                /* Store four blocks.  */
                for (i = 0, u0 = t0, u1 = t1, u2 = t2, u3 = t3; i < 4; i++) {
                        br_aes_ct64_interleave_out(w, q[i], q[4 + i]);
                        le32enc(out + 16*i + 4*0, w[0] ^ u0);
                        le32enc(out + 16*i + 4*1, w[1] ^ u1);
                        le32enc(out + 16*i + 4*2, w[2] ^ u2);
                        le32enc(out + 16*i + 4*3, w[3] ^ u3);
                        aesbear64_xts_update(&u0, &u1, &u2, &u3);
                }

                /* Advance to the next pair of blocks.  */
                t0 = u0, t1 = u1, t2 = u2, t3 = u3;
                in += 64;
                out += 64;
        } while (nbytes -= 64, nbytes);

out:    /* Store the updated tweak.  */
        le32enc(tweak + 4*0, t0);
        le32enc(tweak + 4*1, t1);
        le32enc(tweak + 4*2, t2);
        le32enc(tweak + 4*3, t3);

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static void
aesbear64_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16],
    size_t nbytes, uint8_t auth[static 16], uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];

        KASSERT(nbytes);
        KASSERT(nbytes % 16 == 0);

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);

        /* Initialize garbage blocks.  */
        q[1] = q[2] = q[3] = 0;
        q[5] = q[6] = q[7] = 0;

        /* Load initial authenticator.  */
        w[0] = le32dec(auth + 4*0);
        w[1] = le32dec(auth + 4*1);
        w[2] = le32dec(auth + 4*2);
        w[3] = le32dec(auth + 4*3);

        for (; nbytes; nbytes -= 16, in += 16) {
                /* Combine input block.  */
                w[0] ^= le32dec(in + 4*0);
                w[1] ^= le32dec(in + 4*1);
                w[2] ^= le32dec(in + 4*2);
                w[3] ^= le32dec(in + 4*3);
                br_aes_ct64_interleave_in(&q[0], &q[4], w);

                /* Transform to bitslice, encrypt, transform from bitslice.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                br_aes_ct64_interleave_out(w, q[0], q[4]);
        }

        /* Store updated authenticator.  */
        le32enc(auth + 4*0, w[0]);
        le32enc(auth + 4*1, w[1]);
        le32enc(auth + 4*2, w[2]);
        le32enc(auth + 4*3, w[3]);

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static void
aesbear64_ccm_enc1(const struct aesenc *enc, const uint8_t *in, uint8_t *out,
    size_t nbytes, uint8_t authctr[32], uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];
        uint32_t c0, c1, c2, c3be;

        KASSERT(nbytes);
        KASSERT(nbytes % 16 == 0);

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);

        /* Initialize garbage blocks.  */
        q[2] = q[3] = 0;
        q[6] = q[7] = 0;

        /* Set first block to authenticator.  */
        w[0] = le32dec(authctr + 4*0);
        w[1] = le32dec(authctr + 4*1);
        w[2] = le32dec(authctr + 4*2);
        w[3] = le32dec(authctr + 4*3);

        /* Load initial counter block, big-endian so we can increment it.  */
        c0 = le32dec(authctr + 16 + 4*0);
        c1 = le32dec(authctr + 16 + 4*1);
        c2 = le32dec(authctr + 16 + 4*2);
        c3be = bswap32(le32dec(authctr + 16 + 4*3));

        for (; nbytes; nbytes -= 16, in += 16, out += 16) {
                /* Update authenticator.  */
                w[0] ^= le32dec(in + 4*0);
                w[1] ^= le32dec(in + 4*1);
                w[2] ^= le32dec(in + 4*2);
                w[3] ^= le32dec(in + 4*3);
                br_aes_ct64_interleave_in(&q[0], &q[4], w);

                /* Increment 32-bit counter.  */
                w[0] = c0;
                w[1] = c1;
                w[2] = c2;
                w[3] = bswap32(++c3be);
                br_aes_ct64_interleave_in(&q[1], &q[5], w);

                /* Encrypt authenticator and counter.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);

                /* Encrypt with CTR output.  */
                br_aes_ct64_interleave_out(w, q[1], q[5]);
                le32enc(out + 4*0, le32dec(in + 4*0) ^ w[0]);
                le32enc(out + 4*1, le32dec(in + 4*1) ^ w[1]);
                le32enc(out + 4*2, le32dec(in + 4*2) ^ w[2]);
                le32enc(out + 4*3, le32dec(in + 4*3) ^ w[3]);

                /* Fish out the authenticator so far.  */
                br_aes_ct64_interleave_out(w, q[0], q[4]);
        }

        /* Update authenticator.  */
        le32enc(authctr + 4*0, w[0]);
        le32enc(authctr + 4*1, w[1]);
        le32enc(authctr + 4*2, w[2]);
        le32enc(authctr + 4*3, w[3]);

        /* Update counter.  */
        le32enc(authctr + 16 + 4*3, bswap32(c3be));

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static void
aesbear64_ccm_dec1(const struct aesenc *enc, const uint8_t *in, uint8_t *out,
    size_t nbytes, uint8_t authctr[32], uint32_t nrounds)
{
        uint64_t sk_exp[120];
        uint32_t w[4];
        uint64_t q[8];
        uint32_t c0, c1, c2, c3be;
        uint32_t b0, b1, b2, b3;

        KASSERT(nbytes);
        KASSERT(nbytes % 16 == 0);

        /* Expand round keys for bitslicing.  */
        br_aes_ct64_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);

        /* Initialize garbage blocks.  */
        q[2] = q[3] = 0;
        q[6] = q[7] = 0;

        /* Load initial counter block, big-endian so we can increment it.  */
        c0 = le32dec(authctr + 16 + 4*0);
        c1 = le32dec(authctr + 16 + 4*1);
        c2 = le32dec(authctr + 16 + 4*2);
        c3be = bswap32(le32dec(authctr + 16 + 4*3));

        /* Increment 32-bit counter.  */
        w[0] = c0;
        w[1] = c1;
        w[2] = c2;
        w[3] = bswap32(++c3be);
        br_aes_ct64_interleave_in(&q[1], &q[5], w);

        /*
         * Set the other block to garbage -- we don't have any
         * plaintext to authenticate yet.
         */
        q[0] = q[4] = 0;

        /* Encrypt first CTR.  */
        br_aes_ct64_ortho(q);
        br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
        br_aes_ct64_ortho(q);

        /* Load the initial authenticator.  */
        w[0] = le32dec(authctr + 4*0);
        w[1] = le32dec(authctr + 4*1);
        w[2] = le32dec(authctr + 4*2);
        w[3] = le32dec(authctr + 4*3);
        br_aes_ct64_interleave_in(&q[0], &q[4], w);

        for (;; in += 16, out += 16) {
                /* Decrypt the block.  */
                br_aes_ct64_interleave_out(w, q[1], q[5]);
                b0 = le32dec(in + 4*0) ^ w[0];
                b1 = le32dec(in + 4*1) ^ w[1];
                b2 = le32dec(in + 4*2) ^ w[2];
                b3 = le32dec(in + 4*3) ^ w[3];

                /* Update authenticator.  */
                br_aes_ct64_interleave_out(w, q[0], q[4]);
                w[0] ^= b0;
                w[1] ^= b1;
                w[2] ^= b2;
                w[3] ^= b3;
                br_aes_ct64_interleave_in(&q[0], &q[4], w);

                /* Store plaintext.  */
                le32enc(out + 4*0, b0);
                le32enc(out + 4*1, b1);
                le32enc(out + 4*2, b2);
                le32enc(out + 4*3, b3);

                /* If this is the last block, stop.  */
                if ((nbytes -= 16) == 0)
                        break;

                /* Increment 32-bit counter.  */
                w[0] = c0;
                w[1] = c1;
                w[2] = c2;
                w[3] = bswap32(++c3be);
                br_aes_ct64_interleave_in(&q[1], &q[5], w);

                /* Authenticate previous plaintext, encrypt next CTR.  */
                br_aes_ct64_ortho(q);
                br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
                br_aes_ct64_ortho(q);
        }

        /*
         * Authenticate last plaintext.  We're only doing this for the
         * authenticator, not for the counter, so don't bother to
         * initialize q[2*i].  (Even for the sake of sanitizers,
         * they're already initialized to something by now.)
         */
        br_aes_ct64_ortho(q);
        br_aes_ct64_bitslice_encrypt(nrounds, sk_exp, q);
        br_aes_ct64_ortho(q);

        /* Update authenticator.  */
        br_aes_ct64_interleave_out(w, q[0], q[4]);
        le32enc(authctr + 4*0, w[0]);
        le32enc(authctr + 4*1, w[1]);
        le32enc(authctr + 4*2, w[2]);
        le32enc(authctr + 4*3, w[3]);

        /* Update counter.  */
        le32enc(authctr + 16 + 4*3, bswap32(c3be));

        /* Paranoia: Zero temporary buffers.  */
        explicit_memset(sk_exp, 0, sizeof sk_exp);
        explicit_memset(q, 0, sizeof q);
}

static int
aesbear64_probe(void)
{

        if (aesbear64_xts_update_selftest())
                return -1;

        /* XXX test br_aes_ct64_bitslice_decrypt */
        /* XXX test br_aes_ct64_bitslice_encrypt */
        /* XXX test br_aes_ct64_keysched */
        /* XXX test br_aes_ct64_ortho */
        /* XXX test br_aes_ct64_skey_expand */

        return 0;
}

struct aes_impl aes_bear64_impl = {
        .ai_name = "BearSSL aes_ct64",
        .ai_probe = aesbear64_probe,
        .ai_setenckey = aesbear64_setenckey,
        .ai_setdeckey = aesbear64_setdeckey,
        .ai_enc = aesbear64_enc,
        .ai_dec = aesbear64_dec,
        .ai_cbc_enc = aesbear64_cbc_enc,
        .ai_cbc_dec = aesbear64_cbc_dec,
        .ai_xts_enc = aesbear64_xts_enc,
        .ai_xts_dec = aesbear64_xts_dec,
        .ai_cbcmac_update1 = aesbear64_cbcmac_update1,
        .ai_ccm_enc1 = aesbear64_ccm_enc1,
        .ai_ccm_dec1 = aesbear64_ccm_dec1,
};