/* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
/*      $NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $        */

/*
 * Copyright (c) 2019 Internet Initiative Japan, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
#if 0
__KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
#endif

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/firmware.h>
#include <sys/limits.h>
#include <sys/systm.h>

#include <machine/bus.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include "qatreg.h"
#include "qatvar.h"
#include "qat_aevar.h"

static int      qat_ae_write_4(struct qat_softc *, u_char, bus_size_t,
                    uint32_t);
static int      qat_ae_read_4(struct qat_softc *, u_char, bus_size_t,
                    uint32_t *);
static void     qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t,
                    bus_size_t, uint32_t);
static int      qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t,
                    bus_size_t, uint32_t *);

static u_short  qat_aereg_get_10bit_addr(enum aereg_type, u_short);
static int      qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char,
                    enum aereg_type, u_short, uint32_t);
static int      qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char,
                    enum aereg_type, u_short, uint32_t *);
static int      qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char,
                    enum aereg_type, u_short, uint32_t);
static int      qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char,
                    enum aereg_type, u_short, uint32_t);
static int      qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char,
                    enum aereg_type, u_short, uint32_t);
static int      qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short,
                    u_short *, u_char *);
static int      qat_aereg_abs_data_write(struct qat_softc *, u_char,
                    enum aereg_type, u_short, uint32_t);

static void     qat_ae_enable_ctx(struct qat_softc *, u_char, u_int);
static void     qat_ae_disable_ctx(struct qat_softc *, u_char, u_int);
static void     qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char);
static void     qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char);
static void     qat_ae_write_lm_mode(struct qat_softc *, u_char,
                    enum aereg_type, u_char);
static void     qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char,
                    u_char);
static void     qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char);
static int      qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int,
                    u_int);

static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char);
static int      qat_ae_is_active(struct qat_softc *, u_char);
static int      qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int);

static int      qat_ae_clear_reset(struct qat_softc *);
static int      qat_ae_check(struct qat_softc *);
static int      qat_ae_reset_timestamp(struct qat_softc *);
static void     qat_ae_clear_xfer(struct qat_softc *);
static int      qat_ae_clear_gprs(struct qat_softc *);

static void     qat_ae_get_shared_ustore_ae(u_char, u_char *);
static u_int    qat_ae_ucode_parity64(uint64_t);
static uint64_t qat_ae_ucode_set_ecc(uint64_t);
static int      qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int,
                    const uint64_t *);
static int      qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int,
                    uint64_t *);
static u_int    qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *);
static int      qat_ae_exec_ucode(struct qat_softc *, u_char, u_char,
                    uint64_t *, u_int, int, u_int, u_int *);
static int      qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char,
                    int *, uint64_t *, u_int,
                    u_int *, u_int *, u_int *, u_int *, u_int *);
static int      qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char,
                    u_int, u_int, u_int, u_int, u_int);
static int      qat_ae_get_inst_num(int);
static int      qat_ae_batch_put_lm(struct qat_softc *, u_char,
                    struct qat_ae_batch_init_list *, size_t);
static int      qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int);

static u_int    qat_aefw_csum(char *, int);
static const char *qat_aefw_uof_string(struct qat_softc *, size_t);
static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *,
                    const char *, struct uof_chunk_hdr *);

static int      qat_aefw_load_mof(struct qat_softc *);
static void     qat_aefw_unload_mof(struct qat_softc *);
static int      qat_aefw_load_mmp(struct qat_softc *);
static void     qat_aefw_unload_mmp(struct qat_softc *);

static int      qat_aefw_mof_find_uof0(struct qat_softc *,
                    struct mof_uof_hdr *, struct mof_uof_chunk_hdr *,
                    u_int, size_t, const char *,
                    size_t *, void **);
static int      qat_aefw_mof_find_uof(struct qat_softc *);
static int      qat_aefw_mof_parse(struct qat_softc *);

static int      qat_aefw_uof_parse_image(struct qat_softc *,
                    struct qat_uof_image *, struct uof_chunk_hdr *uch);
static int      qat_aefw_uof_parse_images(struct qat_softc *);
static int      qat_aefw_uof_parse(struct qat_softc *);

static int      qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t,
                    struct qat_dmamem *);
static int      qat_aefw_auth(struct qat_softc *, struct qat_dmamem *);
static int      qat_aefw_suof_load(struct qat_softc *sc,
                    struct qat_dmamem *dma);
static int      qat_aefw_suof_parse_image(struct qat_softc *,
                    struct qat_suof_image *, struct suof_chunk_hdr *);
static int      qat_aefw_suof_parse(struct qat_softc *);
static int      qat_aefw_suof_write(struct qat_softc *);

static int      qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *,
                    struct qat_uof_image *);
static int      qat_aefw_uof_init_ae(struct qat_softc *, u_char);
static int      qat_aefw_uof_init(struct qat_softc *);

static int      qat_aefw_init_memory_one(struct qat_softc *,
                    struct uof_init_mem *);
static void     qat_aefw_free_lm_init(struct qat_softc *, u_char);
static int      qat_aefw_init_ustore(struct qat_softc *);
static int      qat_aefw_init_reg(struct qat_softc *, u_char, u_char,
                    enum aereg_type, u_short, u_int);
static int      qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char,
                    struct qat_uof_image *);
static int      qat_aefw_init_memory(struct qat_softc *);
static int      qat_aefw_init_globals(struct qat_softc *);
static uint64_t qat_aefw_get_uof_inst(struct qat_softc *,
                    struct qat_uof_page *, u_int);
static int      qat_aefw_do_pagein(struct qat_softc *, u_char,
                    struct qat_uof_page *);
static int      qat_aefw_uof_write_one(struct qat_softc *,
                    struct qat_uof_image *);
static int      qat_aefw_uof_write(struct qat_softc *);

static int
qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
        uint32_t value)
{
        int times = TIMEOUT_AE_CSR;

        do {
                qat_ae_local_write_4(sc, ae, offset, value);
                if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
                    LOCAL_CSR_STATUS_STATUS) == 0)
                        return 0;

        } while (times--);

        device_printf(sc->sc_dev,
            "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
        return EFAULT;
}

static int
qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
        uint32_t *value)
{
        int times = TIMEOUT_AE_CSR;
        uint32_t v;

        do {
                v = qat_ae_local_read_4(sc, ae, offset);
                if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
                    LOCAL_CSR_STATUS_STATUS) == 0) {
                        *value = v;
                        return 0;
                }
        } while (times--);

        device_printf(sc->sc_dev,
            "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
        return EFAULT;
}

static void
qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask,
    bus_size_t offset, uint32_t value)
{
        int ctx;
        uint32_t ctxptr;

        MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
            offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
            offset == CTX_STS_INDIRECT ||
            offset == CTX_WAKEUP_EVENTS_INDIRECT ||
            offset == CTX_SIG_EVENTS_INDIRECT ||
            offset == LM_ADDR_0_INDIRECT ||
            offset == LM_ADDR_1_INDIRECT ||
            offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
            offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);

        qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
        for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
                if ((ctx_mask & (1 << ctx)) == 0)
                        continue;
                qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);
                qat_ae_write_4(sc, ae, offset, value);
        }
        qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);
}

static int
qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx,
    bus_size_t offset, uint32_t *value)
{
        int error;
        uint32_t ctxptr;

        MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
            offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
            offset == CTX_STS_INDIRECT ||
            offset == CTX_WAKEUP_EVENTS_INDIRECT ||
            offset == CTX_SIG_EVENTS_INDIRECT ||
            offset == LM_ADDR_0_INDIRECT ||
            offset == LM_ADDR_1_INDIRECT ||
            offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
            offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);

        /* save the ctx ptr */
        qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
        if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
            (ctx & CSR_CTX_POINTER_CONTEXT))
                qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);

        error = qat_ae_read_4(sc, ae, offset, value);

        /* restore ctx ptr */
        if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
            (ctx & CSR_CTX_POINTER_CONTEXT))
                qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);

        return error;
}

static u_short
qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg)
{
        u_short addr;

        switch (regtype) {
        case AEREG_GPA_ABS:
        case AEREG_GPB_ABS:
                addr = (reg & 0x7f) | 0x80;
                break;
        case AEREG_GPA_REL:
        case AEREG_GPB_REL:
                addr = reg & 0x1f;
                break;
        case AEREG_SR_RD_REL:
        case AEREG_SR_WR_REL:
        case AEREG_SR_REL:
                addr = 0x180 | (reg & 0x1f);
                break;
        case AEREG_SR_INDX:
                addr = 0x140 | ((reg & 0x3) << 1);
                break;
        case AEREG_DR_RD_REL:
        case AEREG_DR_WR_REL:
        case AEREG_DR_REL:
                addr = 0x1c0 | (reg & 0x1f);
                break;
        case AEREG_DR_INDX:
                addr = 0x100 | ((reg & 0x3) << 1);
                break;
        case AEREG_NEIGH_INDX:
                addr = 0x241 | ((reg & 0x3) << 1);
                break;
        case AEREG_NEIGH_REL:
                addr = 0x280 | (reg & 0x1f);
                break;
        case AEREG_LMEM0:
                addr = 0x200;
                break;
        case AEREG_LMEM1:
                addr = 0x220;
                break;
        case AEREG_NO_DEST:
                addr = 0x300 | (reg & 0xff);
                break;
        default:
                addr = AEREG_BAD_REGADDR;
                break;
        }
        return (addr);
}

static int
qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{
        uint16_t srchi, srclo, destaddr, data16hi, data16lo;
        uint64_t inst[] = {
                0x0F440000000ull,       /* immed_w1[reg, val_hi16] */
                0x0F040000000ull,       /* immed_w0[reg, val_lo16] */
                0x0F0000C0300ull,       /* nop */
                0x0E000010000ull        /* ctx_arb[kill] */
        };
        const int ninst = nitems(inst);
        const int imm_w1 = 0, imm_w0 = 1;
        unsigned int ctxen;
        uint16_t mask;

        /* This logic only works for GPRs and LM index registers,
           not NN or XFER registers! */
        MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
            regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);

        if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) {
                /* determine the context mode */
                qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
                if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
                        /* 4-ctx mode */
                        if (ctx & 0x1)
                                return EINVAL;
                        mask = 0x1f;
                } else {
                        /* 8-ctx mode */
                        mask = 0x0f;
                }
                if (relreg & ~mask)
                        return EINVAL;
        }
        if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
            AEREG_BAD_REGADDR) {
                return EINVAL;
        }

        data16lo = 0xffff & value;
        data16hi = 0xffff & (value >> 16);
        srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
                (uint16_t)(0xff & data16hi));
        srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
                (uint16_t)(0xff & data16lo));

        switch (regtype) {
        case AEREG_GPA_REL:     /* A rel source */
                inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
                    ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff);
                inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
                    ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff);
                break;
        default:
                inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
                    ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff);
                inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
                    ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff);
                break;
        }

        return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL);
}

static int
qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t *value)
{
        uint64_t inst, savucode;
        uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi;
        u_int uaddr, ustore_addr;
        int error;
        u_short mask, regaddr;
        u_char nae;

        MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
            regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL ||
            regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL ||
            regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);

        if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) ||
            (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) ||
            (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL))
        {
                /* determine the context mode */
                qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
                if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
                        /* 4-ctx mode */
                        if (ctx & 0x1)
                                return EINVAL;
                        mask = 0x1f;
                } else {
                        /* 8-ctx mode */
                        mask = 0x0f;
                }
                if (relreg & ~mask)
                        return EINVAL;
        }
        if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
            AEREG_BAD_REGADDR) {
                return EINVAL;
        }

        /* instruction -- alu[--, --, B, reg] */
        switch (regtype) {
        case AEREG_GPA_REL:
                /* A rel source */
                inst = 0xA070000000ull | (regaddr & 0x3ff);
                break;
        default:
                inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10));
                break;
        }

        /* backup shared control store bit, and force AE to
         * none-shared mode before executing ucode snippet */
        qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
        if (misc & AE_MISC_CONTROL_SHARE_CS) {
                qat_ae_get_shared_ustore_ae(ae, &nae);
                if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae))
                        return EBUSY;
        }

        nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);

        /* read current context */
        qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
        qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);

        qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
        /* prevent clearing the W1C bits: the breakpoint bit,
        ECC error bit, and Parity error bit */
        ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;

        /* change the context */
        if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO))
                qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
                    ctx & ACTIVE_CTX_STATUS_ACNO);
        /* save a ustore location */
        if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) {
                /* restore AE_MISC_CONTROL csr */
                qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);

                /* restore the context */
                if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
                        qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
                            savctx & ACTIVE_CTX_STATUS_ACNO);
                }
                qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);

                return (error);
        }

        /* turn off ustore parity */
        qat_ae_write_4(sc, ae, CTX_ENABLES,
            ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE));

        /* save ustore-addr csr */
        qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);

        /* write the ALU instruction to ustore, enable ecs bit */
        uaddr = 0 | USTORE_ADDRESS_ECS;

        /* set the uaddress */
        qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
        inst = qat_ae_ucode_set_ecc(inst);

        ulo = (uint32_t)(inst & 0xffffffff);
        uhi = (uint32_t)(inst >> 32);

        qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);

        /* this will auto increment the address */
        qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);

        /* set the uaddress */
        qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);

        /* delay for at least 8 cycles */
        qat_ae_wait_num_cycles(sc, ae, 0x8, 0);

        /* read ALU output -- the instruction should have been executed
        prior to clearing the ECS in putUwords */
        qat_ae_read_4(sc, ae, ALU_OUT, value);

        /* restore ustore-addr csr */
        qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

        /* restore the ustore */
        error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode);

        /* restore the context */
        if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
                qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
                    savctx & ACTIVE_CTX_STATUS_ACNO);
        }

        qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);

        /* restore AE_MISC_CONTROL csr */
        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);

        qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);

        return error;
}

static int
qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{
        bus_size_t addr;
        int error;
        uint32_t ctxen;
        u_short mask;
        u_short dr_offset;

        MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL ||
            regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL);

        error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
        if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
                if (ctx & 0x1) {
                        device_printf(sc->sc_dev,
                            "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx);
                        return EINVAL;
                }
                mask = 0x1f;
                dr_offset = 0x20;

        } else {
                mask = 0x0f;
                dr_offset = 0x10;
        }

        if (relreg & ~mask)
                return EINVAL;

        addr = relreg + (ctx << 0x5);

        switch (regtype) {
        case AEREG_SR_REL:
        case AEREG_SR_RD_REL:
                qat_ae_xfer_write_4(sc, ae, addr, value);
                break;
        case AEREG_DR_REL:
        case AEREG_DR_RD_REL:
                qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value);
                break;
        default:
                error = EINVAL;
        }

        return error;
}

static int
qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{

        panic("notyet");

        return 0;
}

static int
qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{

        panic("notyet");

        return 0;
}

static int
qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae,
        u_short absreg, u_short *relreg, u_char *ctx)
{
        uint32_t ctxen;

        qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
        if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
                /* 4-ctx mode */
                *relreg = absreg & 0x1f;
                *ctx = (absreg >> 0x4) & 0x6;
        } else {
                /* 8-ctx mode */
                *relreg = absreg & 0x0f;
                *ctx = (absreg >> 0x4) & 0x7;
        }

        return 0;
}

static int
qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae,
        enum aereg_type regtype, u_short absreg, uint32_t value)
{
        int error;
        u_short relreg;
        u_char ctx;

        qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx);

        switch (regtype) {
        case AEREG_GPA_ABS:
                MPASS(absreg < MAX_GPR_REG);
                error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL,
                    relreg, value);
                break;
        case AEREG_GPB_ABS:
                MPASS(absreg < MAX_GPR_REG);
                error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL,
                    relreg, value);
                break;
        case AEREG_DR_RD_ABS:
                MPASS(absreg < MAX_XFER_REG);
                error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL,
                    relreg, value);
                break;
        case AEREG_SR_RD_ABS:
                MPASS(absreg < MAX_XFER_REG);
                error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL,
                    relreg, value);
                break;
        case AEREG_DR_WR_ABS:
                MPASS(absreg < MAX_XFER_REG);
                error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL,
                    relreg, value);
                break;
        case AEREG_SR_WR_ABS:
                MPASS(absreg < MAX_XFER_REG);
                error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL,
                    relreg, value);
                break;
        case AEREG_NEIGH_ABS:
                MPASS(absreg < MAX_NN_REG);
                if (absreg >= MAX_NN_REG)
                        return EINVAL;
                error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL,
                    relreg, value);
                break;
        default:
                panic("Invalid Register Type");
        }

        return error;
}

static void
qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
        uint32_t ctxen;

        qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
        ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;

        if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
                ctx_mask &= 0x55;
        } else {
                ctx_mask &= 0xff;
        }

        ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE);
        qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
}

static void
qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
        uint32_t ctxen;

        qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
        ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
        ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE));
        qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
}

static void
qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
        uint32_t val, nval;

        qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
        val &= CTX_ENABLES_IGNORE_W1C_MASK;

        if (mode == 4)
                nval = val | CTX_ENABLES_INUSE_CONTEXTS;
        else
                nval = val & ~CTX_ENABLES_INUSE_CONTEXTS;

        if (val != nval)
                qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

static void
qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
        uint32_t val, nval;

        qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
        val &= CTX_ENABLES_IGNORE_W1C_MASK;

        if (mode)
                nval = val | CTX_ENABLES_NN_MODE;
        else
                nval = val & ~CTX_ENABLES_NN_MODE;

        if (val != nval)
                qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

static void
qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae,
        enum aereg_type lm, u_char mode)
{
        uint32_t val, nval;
        uint32_t bit;

        qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
        val &= CTX_ENABLES_IGNORE_W1C_MASK;

        switch (lm) {
        case AEREG_LMEM0:
                bit = CTX_ENABLES_LMADDR_0_GLOBAL;
                break;
        case AEREG_LMEM1:
                bit = CTX_ENABLES_LMADDR_1_GLOBAL;
                break;
        default:
                panic("invalid lmem reg type");
                break;
        }

        if (mode)
                nval = val | bit;
        else
                nval = val & ~bit;

        if (val != nval)
                qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

static void
qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode)
{
        uint32_t val, nval;

        qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);

        if (mode == 1)
                nval = val | AE_MISC_CONTROL_SHARE_CS;
        else
                nval = val & ~AE_MISC_CONTROL_SHARE_CS;

        if (val != nval)
                qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval);
}

static void
qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
        u_char nae;

        qat_ae_get_shared_ustore_ae(ae, &nae);

        qat_ae_write_shared_cs_mode0(sc, ae, mode);

        if ((sc->sc_ae_mask & (1 << nae))) {
                qat_ae_write_shared_cs_mode0(sc, nae, mode);
        }
}

static int
qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae,
        u_int reload_size, int shared_mode, u_int ustore_dram_addr)
{
        uint32_t val, cs_reload;

        switch (reload_size) {
        case 0:
                cs_reload = 0x0;
                break;
        case QAT_2K:
                cs_reload = 0x1;
                break;
        case QAT_4K:
                cs_reload = 0x2;
                break;
        case QAT_8K:
                cs_reload = 0x3;
                break;
        default:
                return EINVAL;
        }

        if (cs_reload)
                QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr;

        QAT_AE(sc, ae).qae_reload_size = reload_size;

        qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
        val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD |
            AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS);
        val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) |
            __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD);
        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);

        return 0;
}

static enum qat_ae_status
qat_ae_get_status(struct qat_softc *sc, u_char ae)
{
        int error;
        uint32_t val = 0;

        error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
        if (error || val & CTX_ENABLES_ENABLE)
                return QAT_AE_ENABLED;

        qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
        if (val & ACTIVE_CTX_STATUS_ABO)
                return QAT_AE_ACTIVE;

        return QAT_AE_DISABLED;
}


static int
qat_ae_is_active(struct qat_softc *sc, u_char ae)
{
        uint32_t val;

        if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
                return 1;

        qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
        if (val & ACTIVE_CTX_STATUS_ABO)
                return 1;
        else
                return 0;
}

/* returns 1 if actually waited for specified number of cycles */
static int
qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check)
{
        uint32_t cnt, actx;
        int pcnt, ccnt, elapsed, times;

        qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
        pcnt = cnt & 0xffff;

        times = TIMEOUT_AE_CHECK;
        do {
                qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
                ccnt = cnt & 0xffff;

                elapsed = ccnt - pcnt;
                if (elapsed == 0) {
                        times--;
                }
                if (times <= 0) {
                        device_printf(sc->sc_dev,
                            "qat_ae_wait_num_cycles timeout\n");
                        return -1;
                }

                if (elapsed < 0)
                        elapsed += 0x10000;

                if (elapsed >= CYCLES_FROM_READY2EXE && check) {
                        if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS,
                            &actx) == 0) {
                                if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
                                        return 0;
                        }
                }
        } while (cycles > elapsed);

        if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) {
                if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
                        return 0;
        }

        return 1;
}

int
qat_ae_init(struct qat_softc *sc)
{
        int error;
        uint32_t mask, val = 0;
        u_char ae;

        /* XXX adf_initSysMemInfo */

        /* XXX Disable clock gating for some chip if debug mode */

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                struct qat_ae *qae = &sc->sc_ae[ae];
                if (!(mask & 1))
                        continue;

                qae->qae_ustore_size = USTORE_SIZE;

                qae->qae_free_addr = 0;
                qae->qae_free_size = USTORE_SIZE;
                qae->qae_live_ctx_mask = AE_ALL_CTX;
                qae->qae_ustore_dram_addr = 0;
                qae->qae_reload_size = 0;
        }

        /* XXX Enable attention interrupt */

        error = qat_ae_clear_reset(sc);
        if (error)
                return error;

        qat_ae_clear_xfer(sc);

        if (!sc->sc_hw.qhw_fw_auth) {
                error = qat_ae_clear_gprs(sc);
                if (error)
                        return error;
        }

        /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;
                qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val);
                val |= 0x1;
                qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val);
        }

        error = qat_ae_clear_reset(sc);
        if (error)
                return error;

        /* XXX XXX XXX Clean MMP memory if mem scrub is supported */
        /* halMem_ScrubMMPMemory */

        return 0;
}

int
qat_ae_start(struct qat_softc *sc)
{
        int error;
        u_char ae;

        for (ae = 0; ae < sc->sc_ae_num; ae++) {
                if ((sc->sc_ae_mask & (1 << ae)) == 0)
                        continue;

                error = qat_aefw_start(sc, ae, 0xff);
                if (error)
                        return error;
        }

        return 0;
}

void
qat_ae_cluster_intr(void *arg)
{
        /* Nothing to implement until we support SRIOV. */
        printf("qat_ae_cluster_intr\n");
}

static int
qat_ae_clear_reset(struct qat_softc *sc)
{
        int error;
        uint32_t times, reset, clock, reg, mask;
        u_char ae;

        reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
        reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK));
        reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK));
        times = TIMEOUT_AE_RESET;
        do {
                qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset);
                if ((times--) == 0) {
                        device_printf(sc->sc_dev, "couldn't reset AEs\n");
                        return EBUSY;
                }
                reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
        } while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) |
            __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK))
            & reg);

        /* Enable clock for AE and QAT */
        clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN);
        clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK);
        clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK);
        qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock);

        error = qat_ae_check(sc);
        if (error)
                return error;

        /*
         * Set undefined power-up/reset states to reasonable default values...
         * just to make sure we're starting from a known point
         */
        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;

                /* init the ctx_enable */
                qat_ae_write_4(sc, ae, CTX_ENABLES,
                    CTX_ENABLES_INIT);

                /* initialize the PCs */
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
                    CTX_STS_INDIRECT,
                    UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);

                /* init the ctx_arb */
                qat_ae_write_4(sc, ae, CTX_ARB_CNTL,
                    CTX_ARB_CNTL_INIT);

                /* enable cc */
                qat_ae_write_4(sc, ae, CC_ENABLE,
                    CC_ENABLE_INIT);
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
                    CTX_WAKEUP_EVENTS_INDIRECT,
                    CTX_WAKEUP_EVENTS_INDIRECT_INIT);
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
                    CTX_SIG_EVENTS_INDIRECT,
                    CTX_SIG_EVENTS_INDIRECT_INIT);
        }

        if ((sc->sc_ae_mask != 0) &&
            sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) {
                /* XXX XXX XXX init eSram only when this is boot time */
        }

        if ((sc->sc_ae_mask != 0) &&
            sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) {
                /* XXX XXX XXX wait shram to complete initialization */
        }

        qat_ae_reset_timestamp(sc);

        return 0;
}

static int
qat_ae_check(struct qat_softc *sc)
{
        int error, times, ae;
        uint32_t cnt, pcnt, mask;

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;

                times = TIMEOUT_AE_CHECK;
                error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
                if (error) {
                        device_printf(sc->sc_dev,
                            "couldn't access AE %d CSR\n", ae);
                        return error;
                }
                pcnt = cnt & 0xffff;

                while (1) {
                        error = qat_ae_read_4(sc, ae,
                            PROFILE_COUNT, &cnt);
                        if (error) {
                                device_printf(sc->sc_dev,
                                    "couldn't access AE %d CSR\n", ae);
                                return error;
                        }
                        cnt &= 0xffff;
                        if (cnt == pcnt)
                                times--;
                        else
                                break;
                        if (times <= 0) {
                                device_printf(sc->sc_dev,
                                    "AE %d CSR is useless\n", ae);
                                return EFAULT;
                        }
                }
        }

        return 0;
}

static int
qat_ae_reset_timestamp(struct qat_softc *sc)
{
        uint32_t misc, mask;
        u_char ae;

        /* stop the timestamp timers */
        misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC);
        if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) {
                qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
                    misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN));
        }

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;
                qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0);
                qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0);
        }

        /* start timestamp timers */
        qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
            misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN);

        return 0;
}

static void
qat_ae_clear_xfer(struct qat_softc *sc)
{
        u_int mask, reg;
        u_char ae;

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;

                for (reg = 0; reg < MAX_GPR_REG; reg++) {
                        qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS,
                            reg, 0);
                        qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS,
                            reg, 0);
                }
        }
}

static int
qat_ae_clear_gprs(struct qat_softc *sc)
{
        uint32_t val;
        uint32_t saved_ctx = 0;
        int times = TIMEOUT_AE_CHECK, rv;
        u_char ae;
        u_int mask;

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;

                /* turn off share control store bit */
                val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
                val &= ~AE_MISC_CONTROL_SHARE_CS;
                qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);

                /* turn off ucode parity */
                /* make sure nn_mode is set to self */
                qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
                val &= CTX_ENABLES_IGNORE_W1C_MASK;
                val |= CTX_ENABLES_NN_MODE;
                val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE;
                qat_ae_write_4(sc, ae, CTX_ENABLES, val);

                /* copy instructions to ustore */
                qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst),
                    ae_clear_gprs_inst);

                /* set PC */
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT,
                    UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);

                /* save current context */
                qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx);
                /* change the active context */
                /* start the context from ctx 0 */
                qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0);

                /* wakeup-event voluntary */
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
                    CTX_WAKEUP_EVENTS_INDIRECT,
                    CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);
                /* clean signals */
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
                    CTX_SIG_EVENTS_INDIRECT, 0);
                qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);

                qat_ae_enable_ctx(sc, ae, AE_ALL_CTX);
        }

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;
                /* wait for AE to finish */
                do {
                        rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1);
                } while (rv && times--);
                if (times <= 0) {
                        device_printf(sc->sc_dev,
                            "qat_ae_clear_gprs timeout");
                        return ETIMEDOUT;
                }
                qat_ae_disable_ctx(sc, ae, AE_ALL_CTX);
                /* change the active context */
                qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
                    saved_ctx & ACTIVE_CTX_STATUS_ACNO);
                /* init the ctx_enable */
                qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT);
                /* initialize the PCs */
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
                    CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
                /* init the ctx_arb */
                qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT);
                /* enable cc */
                qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT);
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
                    CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT);
                qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT,
                    CTX_SIG_EVENTS_INDIRECT_INIT);
        }

        return 0;
}

static void
qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae)
{
        if (ae & 0x1)
                *nae = ae - 1;
        else
                *nae = ae + 1;
}

static u_int
qat_ae_ucode_parity64(uint64_t ucode)
{

        ucode ^= ucode >> 1;
        ucode ^= ucode >> 2;
        ucode ^= ucode >> 4;
        ucode ^= ucode >> 8;
        ucode ^= ucode >> 16;
        ucode ^= ucode >> 32;

        return ((u_int)(ucode & 1));
}

static uint64_t
qat_ae_ucode_set_ecc(uint64_t ucode)
{
        static const uint64_t
                bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL,
                bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL,
                bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL,
                bit6mask=0xdaf69a46910ULL;

        /* clear the ecc bits */
        ucode &= ~(0x7fULL << USTORE_ECC_BIT_0);

        ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) <<
            USTORE_ECC_BIT_0;
        ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) <<
            USTORE_ECC_BIT_1;
        ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) <<
            USTORE_ECC_BIT_2;
        ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) <<
            USTORE_ECC_BIT_3;
        ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) <<
            USTORE_ECC_BIT_4;
        ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) <<
            USTORE_ECC_BIT_5;
        ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) <<
            USTORE_ECC_BIT_6;

        return (ucode);
}

static int
qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
        const uint64_t *ucode)
{
        uint64_t tmp;
        uint32_t ustore_addr, ulo, uhi;
        int i;

        qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
        uaddr |= USTORE_ADDRESS_ECS;

        qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
        for (i = 0; i < ninst; i++) {
                tmp = qat_ae_ucode_set_ecc(ucode[i]);
                ulo = (uint32_t)(tmp & 0xffffffff);
                uhi = (uint32_t)(tmp >> 32);

                qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);
                /* this will auto increment the address */
                qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);
        }
        qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

        return 0;
}

static int
qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
    uint64_t *ucode)
{
        uint32_t misc, ustore_addr, ulo, uhi;
        u_int ii;
        u_char nae;

        if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
                return EBUSY;

        /* determine whether it neighbour AE runs in shared control store
         * status */
        qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
        if (misc & AE_MISC_CONTROL_SHARE_CS) {
                qat_ae_get_shared_ustore_ae(ae, &nae);
                if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
                        return EBUSY;
        }

        /* if reloadable, then get it all from dram-ustore */
        if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD))
                panic("notyet"); /* XXX getReloadUwords */

        /* disable SHARE_CS bit to workaround silicon bug */
        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb);

        MPASS(uaddr + ninst <= USTORE_SIZE);

        /* save ustore-addr csr */
        qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);

        uaddr |= USTORE_ADDRESS_ECS;    /* enable ecs bit */
        for (ii = 0; ii < ninst; ii++) {
                qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);

                uaddr++;
                qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo);
                qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi);
                ucode[ii] = uhi;
                ucode[ii] = (ucode[ii] << 32) | ulo;
        }

        /* restore SHARE_CS bit to workaround silicon bug */
        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
        qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

        return 0;
}

static u_int
qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr,
    u_int *value)
{
        const uint64_t *inst_arr;
        u_int ninst0, curvalue;
        int ii, vali, fixup, usize = 0;

        if (size == 0)
                return 0;

        ninst0 = ninst;
        vali = 0;
        curvalue = value[vali++];

        switch (size) {
        case 0x1:
                inst_arr = ae_inst_1b;
                usize = nitems(ae_inst_1b);
                break;
        case 0x2:
                inst_arr = ae_inst_2b;
                usize = nitems(ae_inst_2b);
                break;
        case 0x3:
                inst_arr = ae_inst_3b;
                usize = nitems(ae_inst_3b);
                break;
        default:
                inst_arr = ae_inst_4b;
                usize = nitems(ae_inst_4b);
                break;
        }

        fixup = ninst;
        for (ii = 0; ii < usize; ii++)
                ucode[ninst++] = inst_arr[ii];

        INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr));
        fixup++;
        INSERT_IMMED_GPRA_CONST(ucode[fixup], 0);
        fixup++;
        INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
        fixup++;
        INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
        /* XXX fixup++ ? */

        if (size <= 0x4)
                return (ninst - ninst0);

        size -= sizeof(u_int);
        while (size >= sizeof(u_int)) {
                curvalue = value[vali++];
                fixup = ninst;
                ucode[ninst++] = ae_inst_4b[0x2];
                ucode[ninst++] = ae_inst_4b[0x3];
                ucode[ninst++] = ae_inst_4b[0x8];
                INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
                fixup++;
                INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
                /* XXX fixup++ ? */

                addr += sizeof(u_int);
                size -= sizeof(u_int);
        }
        /* call this function recusive when the left size less than 4 */
        ninst +=
            qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali);

        return (ninst - ninst0);
}

static int
qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx,
    uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles,
    u_int *endpc)
{
        int error = 0, share_cs = 0;
        uint64_t savucode[MAX_EXEC_INST];
        uint32_t indr_lm_addr_0, indr_lm_addr_1;
        uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1;
        uint32_t indr_future_cnt_sig;
        uint32_t indr_sig, active_sig;
        uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl;
        uint32_t misc, nmisc, ctxen;
        u_char nae;

        MPASS(ninst <= USTORE_SIZE);

        if (qat_ae_is_active(sc, ae))
                return EBUSY;

        /* save current LM addr */
        qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0);
        qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1);
        qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
            &indr_lm_addr_byte_0);
        qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
            &indr_lm_addr_byte_1);

        /* backup shared control store bit, and force AE to
           none-shared mode before executing ucode snippet */
        qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
        if (misc & AE_MISC_CONTROL_SHARE_CS) {
                share_cs = 1;
                qat_ae_get_shared_ustore_ae(ae, &nae);
                if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
                        return EBUSY;
        }
        nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);

        /* save current states: */
        if (ninst <= MAX_EXEC_INST) {
                error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode);
                if (error) {
                        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
                        return error;
                }
        }

        /* save wakeup-events */
        qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT,
            &wakeup_ev);
        /* save PC */
        qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc);
        savpc &= UPC_MASK;

        /* save ctx enables */
        qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
        ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
        /* save conditional-code */
        qat_ae_read_4(sc, ae, CC_ENABLE, &savcc);
        /* save current context */
        qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
        qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);

        /* save indirect csrs */
        qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
            &indr_future_cnt_sig);
        qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig);
        qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig);

        /* turn off ucode parity */
        qat_ae_write_4(sc, ae, CTX_ENABLES,
            ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE);

        /* copy instructions to ustore */
        qat_ae_ucode_write(sc, ae, 0, ninst, ucode);
        /* set PC */
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0);
        /* change the active context */
        qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
            ctx & ACTIVE_CTX_STATUS_ACNO);

        if (cond_code_off) {
                /* disable conditional-code*/
                qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff);
        }

        /* wakeup-event voluntary */
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx,
            CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);

        /* clean signals */
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0);
        qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);

        /* enable context */
        qat_ae_enable_ctx(sc, ae, 1 << ctx);

        /* wait for it to finish */
        if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0)
                error = ETIMEDOUT;

        /* see if we need to get the current PC */
        if (endpc != NULL) {
                uint32_t ctx_status;

                qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
                    &ctx_status);
                *endpc = ctx_status & UPC_MASK;
        }
#if 0
        {
                uint32_t ctx_status;

                qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
                    &ctx_status);
                printf("%s: endpc 0x%08x\n", __func__,
                    ctx_status & UPC_MASK);
        }
#endif

        /* retore to previous states: */
        /* disable context */
        qat_ae_disable_ctx(sc, ae, 1 << ctx);
        if (ninst <= MAX_EXEC_INST) {
                /* instructions */
                qat_ae_ucode_write(sc, ae, 0, ninst, savucode);
        }
        /* wakeup-events */
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT,
            wakeup_ev);
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc);

        /* only restore shared control store bit,
           other bit might be changed by AE code snippet */
        qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
        if (share_cs)
                nmisc = misc | AE_MISC_CONTROL_SHARE_CS;
        else
                nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
        qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
        /* conditional-code */
        qat_ae_write_4(sc, ae, CC_ENABLE, savcc);
        /* change the active context */
        qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
            savctx & ACTIVE_CTX_STATUS_ACNO);
        /* restore the nxt ctx to run */
        qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
        /* restore current LM addr */
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT,
            indr_lm_addr_0);
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT,
            indr_lm_addr_1);
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
            indr_lm_addr_byte_0);
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
            indr_lm_addr_byte_1);

        /* restore indirect csrs */
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
            indr_future_cnt_sig);
        qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT,
            indr_sig);
        qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig);

        /* ctx-enables */
        qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);

        return error;
}

static int
qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx,
    int *first_exec, uint64_t *ucode, u_int ninst,
    u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1)
{

        if (*first_exec) {
                qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
                qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
                qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
                qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
                qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);
                *first_exec = 0;
        }

        return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL);
}

static int
qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx,
    u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1)
{
        qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
        qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
        qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
        qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
        qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);

        return 0;
}

static int
qat_ae_get_inst_num(int lmsize)
{
        int ninst, left;

        if (lmsize == 0)
                return 0;

        left = lmsize % sizeof(u_int);

        if (left) {
                ninst = nitems(ae_inst_1b) +
                    qat_ae_get_inst_num(lmsize - left);
        } else {
                /* 3 instruction is needed for further code */
                ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b);
        }

        return (ninst);
}

static int
qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae,
    struct qat_ae_batch_init_list *qabi_list, size_t nqabi)
{
        struct qat_ae_batch_init *qabi;
        size_t alloc_ninst, ninst;
        uint64_t *ucode;
        u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1;
        int insnsz, error = 0, execed = 0, first_exec = 1;

        if (STAILQ_FIRST(qabi_list) == NULL)
                return 0;

        alloc_ninst = min(USTORE_SIZE, nqabi);
        ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst);

        ninst = 0;
        STAILQ_FOREACH(qabi, qabi_list, qabi_next) {
                insnsz = qat_ae_get_inst_num(qabi->qabi_size);
                if (insnsz + ninst > alloc_ninst) {
                        /* add ctx_arb[kill] */
                        ucode[ninst++] = 0x0E000010000ull;
                        execed = 1;

                        error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
                            &first_exec, ucode, ninst,
                            &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
                        if (error) {
                                qat_ae_restore_init_lm_gprs(sc, ae, 0,
                                    gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
                                qat_free_mem(ucode);
                                return error;
                        }
                        /* run microExec to execute the microcode */
                        ninst = 0;
                }
                ninst += qat_ae_concat_ucode(ucode, ninst,
                    qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value);
        }

        if (ninst > 0) {
                ucode[ninst++] = 0x0E000010000ull;
                execed = 1;

                error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
                    &first_exec, ucode, ninst,
                    &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
        }
        if (execed) {
                qat_ae_restore_init_lm_gprs(sc, ae, 0,
                    gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
        }

        qat_free_mem(ucode);

        return error;
}

static int
qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc)
{

        if (qat_ae_is_active(sc, ae))
                return EBUSY;

        qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT,
            UPC_MASK & upc);
        return 0;
}

static inline u_int
qat_aefw_csum_calc(u_int reg, int ch)
{
        int i;
        u_int topbit = CRC_BITMASK(CRC_WIDTH - 1);
        u_int inbyte = (u_int)((reg >> 0x18) ^ ch);

        reg ^= inbyte << (CRC_WIDTH - 0x8);
        for (i = 0; i < 0x8; i++) {
                if (reg & topbit)
                        reg = (reg << 1) ^ CRC_POLY;
                else
                        reg <<= 1;
        }

        return (reg & CRC_WIDTHMASK(CRC_WIDTH));
}

static u_int
qat_aefw_csum(char *buf, int size)
{
        u_int csum = 0;

        while (size--) {
                csum = qat_aefw_csum_calc(csum, *buf++);
        }

        return csum;
}

static const char *
qat_aefw_uof_string(struct qat_softc *sc, size_t offset)
{
        if (offset >= sc->sc_aefw_uof.qafu_str_tab_size)
                return NULL;
        if (sc->sc_aefw_uof.qafu_str_tab == NULL)
                return NULL;

        return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset);
}

static struct uof_chunk_hdr *
qat_aefw_uof_find_chunk(struct qat_softc *sc,
        const char *id, struct uof_chunk_hdr *cur)
{
        struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr;
        struct uof_chunk_hdr *uch;
        int i;

        uch = (struct uof_chunk_hdr *)(uoh + 1);
        for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) {
                if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size)
                        return NULL;

                if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN))
                        return uch;
        }

        return NULL;
}

static int
qat_aefw_load_mof(struct qat_softc *sc)
{
        const struct firmware *fw;

        fw = firmware_get(sc->sc_hw.qhw_mof_fwname);
        if (fw == NULL) {
                device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
                    sc->sc_hw.qhw_mof_fwname);
                return ENXIO;
        }

        sc->sc_fw_mof = qat_alloc_mem(fw->datasize);
        sc->sc_fw_mof_size = fw->datasize;
        memcpy(sc->sc_fw_mof, fw->data, fw->datasize);
        firmware_put(fw, FIRMWARE_UNLOAD);
        return 0;
}

static void
qat_aefw_unload_mof(struct qat_softc *sc)
{
        if (sc->sc_fw_mof != NULL) {
                qat_free_mem(sc->sc_fw_mof);
                sc->sc_fw_mof = NULL;
        }
}

static int
qat_aefw_load_mmp(struct qat_softc *sc)
{
        const struct firmware *fw;

        fw = firmware_get(sc->sc_hw.qhw_mmp_fwname);
        if (fw == NULL) {
                device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
                    sc->sc_hw.qhw_mmp_fwname);
                return ENXIO;
        }

        sc->sc_fw_mmp = qat_alloc_mem(fw->datasize);
        sc->sc_fw_mmp_size = fw->datasize;
        memcpy(sc->sc_fw_mmp, fw->data, fw->datasize);
        firmware_put(fw, FIRMWARE_UNLOAD);
        return 0;
}

static void
qat_aefw_unload_mmp(struct qat_softc *sc)
{
        if (sc->sc_fw_mmp != NULL) {
                qat_free_mem(sc->sc_fw_mmp);
                sc->sc_fw_mmp = NULL;
        }
}

static int
qat_aefw_mof_find_uof0(struct qat_softc *sc,
        struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head,
        u_int nchunk, size_t size, const char *id,
        size_t *fwsize, void **fwptr)
{
        int i;
        char *uof_name;

        for (i = 0; i < nchunk; i++) {
                struct mof_uof_chunk_hdr *much = &head[i];

                if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN))
                        return EINVAL;

                if (much->much_offset + much->much_size > size)
                        return EINVAL;

                if (sc->sc_mof.qmf_sym_size <= much->much_name)
                        return EINVAL;

                uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym +
                    much->much_name);

                if (!strcmp(uof_name, sc->sc_fw_uof_name)) {
                        *fwptr = (void *)((uintptr_t)muh +
                            (uintptr_t)much->much_offset);
                        *fwsize = (size_t)much->much_size;
                        return 0;
                }
        }

        return ENOENT;
}

static int
qat_aefw_mof_find_uof(struct qat_softc *sc)
{
        struct mof_uof_hdr *uof_hdr, *suof_hdr;
        u_int nuof_chunks = 0, nsuof_chunks = 0;
        int error;

        uof_hdr = sc->sc_mof.qmf_uof_objs;
        suof_hdr = sc->sc_mof.qmf_suof_objs;

        if (uof_hdr != NULL) {
                if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) {
                        return EINVAL;
                }
                nuof_chunks = uof_hdr->muh_num_chunks;
        }
        if (suof_hdr != NULL) {
                if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks)
                        return EINVAL;
                nsuof_chunks = suof_hdr->muh_num_chunks;
        }

        if (nuof_chunks + nsuof_chunks == 0)
                return EINVAL;

        if (uof_hdr != NULL) {
                error = qat_aefw_mof_find_uof0(sc, uof_hdr,
                    (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks,
                    sc->sc_mof.qmf_uof_objs_size, UOF_IMAG,
                    &sc->sc_fw_uof_size, &sc->sc_fw_uof);
                if (error && error != ENOENT)
                        return error;
        }

        if (suof_hdr != NULL) {
                error = qat_aefw_mof_find_uof0(sc, suof_hdr,
                    (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks,
                    sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG,
                    &sc->sc_fw_suof_size, &sc->sc_fw_suof);
                if (error && error != ENOENT)
                        return error;
        }

        if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL)
                return ENOENT;

        return 0;
}

static int
qat_aefw_mof_parse(struct qat_softc *sc)
{
        const struct mof_file_hdr *mfh;
        const struct mof_file_chunk_hdr *mfch;
        size_t size;
        u_int csum;
        int error, i;

        size = sc->sc_fw_mof_size;

        if (size < sizeof(struct mof_file_hdr))
                return EINVAL;
        size -= sizeof(struct mof_file_hdr);

        mfh = sc->sc_fw_mof;

        if (mfh->mfh_fid != MOF_FID)
                return EINVAL;

        csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof +
            offsetof(struct mof_file_hdr, mfh_min_ver)),
            sc->sc_fw_mof_size -
            offsetof(struct mof_file_hdr, mfh_min_ver));
        if (mfh->mfh_csum != csum)
                return EINVAL;

        if (mfh->mfh_min_ver != MOF_MIN_VER ||
            mfh->mfh_maj_ver != MOF_MAJ_VER)
                return EINVAL;

        if (mfh->mfh_max_chunks < mfh->mfh_num_chunks)
                return EINVAL;

        if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks)
                return EINVAL;
        mfch = (const struct mof_file_chunk_hdr *)(mfh + 1);

        for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) {
                if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size)
                        return EINVAL;

                if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) {
                        if (sc->sc_mof.qmf_sym != NULL)
                                return EINVAL;

                        sc->sc_mof.qmf_sym =
                            (void *)((uintptr_t)sc->sc_fw_mof +
                            (uintptr_t)mfch->mfch_offset + sizeof(u_int));
                        sc->sc_mof.qmf_sym_size =
                            *(u_int *)((uintptr_t)sc->sc_fw_mof +
                            (uintptr_t)mfch->mfch_offset);

                        if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0)
                                return EINVAL;
                        if (mfch->mfch_size != sc->sc_mof.qmf_sym_size +
                            sizeof(u_int) || mfch->mfch_size == 0)
                                return EINVAL;
                        if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym +
                            sc->sc_mof.qmf_sym_size - 1) != '\0')
                                return EINVAL;

                } else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) {
                        if (sc->sc_mof.qmf_uof_objs != NULL)
                                return EINVAL;

                        sc->sc_mof.qmf_uof_objs =
                            (void *)((uintptr_t)sc->sc_fw_mof +
                            (uintptr_t)mfch->mfch_offset);
                        sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size;

                } else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) {
                        if (sc->sc_mof.qmf_suof_objs != NULL)
                                return EINVAL;

                        sc->sc_mof.qmf_suof_objs =
                            (void *)((uintptr_t)sc->sc_fw_mof +
                            (uintptr_t)mfch->mfch_offset);
                        sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size;
                }
        }

        if (sc->sc_mof.qmf_sym == NULL ||
            (sc->sc_mof.qmf_uof_objs == NULL &&
            sc->sc_mof.qmf_suof_objs == NULL))
                return EINVAL;

        error = qat_aefw_mof_find_uof(sc);
        if (error)
                return error;
        return 0;
}

static int
qat_aefw_uof_parse_image(struct qat_softc *sc,
        struct qat_uof_image *qui, struct uof_chunk_hdr *uch)
{
        struct uof_image *image;
        struct uof_code_page *page;
        uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;
        size_t lim = uch->uch_offset + uch->uch_size, size;
        int i, p;

        size = uch->uch_size;
        if (size < sizeof(struct uof_image))
                return EINVAL;
        size -= sizeof(struct uof_image);

        qui->qui_image = image =
            (struct uof_image *)(base + uch->uch_offset);

#define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim)                 \
do {                                                                    \
        u_int nent;                                                     \
        nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\
        if ((lim) < off + sizeof(struct uof_obj_table) +                \
            sizeof(type) * nent)                                        \
                return EINVAL;                                          \
        *(np) = nent;                                                   \
        if (nent > 0)                                                   \
                *(typep) = (type)((struct uof_obj_table *)              \
                    ((base) + (off)) + 1);                              \
        else                                                            \
                *(typep) = NULL;                                        \
} while (0)

        ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg,
            struct uof_ae_reg *, base, image->ui_reg_tab, lim);
        ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym,
            struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim);
        ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak,
            struct qui_sbreak *, base, image->ui_sbreak_tab, lim);

        if (size < sizeof(struct uof_code_page) * image->ui_num_pages)
                return EINVAL;
        if (nitems(qui->qui_pages) < image->ui_num_pages)
                return EINVAL;

        page = (struct uof_code_page *)(image + 1);

        for (p = 0; p < image->ui_num_pages; p++, page++) {
                struct qat_uof_page *qup = &qui->qui_pages[p];
                struct uof_code_area *uca;

                qup->qup_page_num = page->ucp_page_num;
                qup->qup_def_page = page->ucp_def_page;
                qup->qup_page_region = page->ucp_page_region;
                qup->qup_beg_vaddr = page->ucp_beg_vaddr;
                qup->qup_beg_paddr = page->ucp_beg_paddr;

                ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var,
                    struct uof_uword_fixup *, base,
                    page->ucp_uc_var_tab, lim);
                ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var,
                    struct uof_import_var *, base,
                    page->ucp_imp_var_tab, lim);
                ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr,
                    struct uof_uword_fixup *, base,
                    page->ucp_imp_expr_tab, lim);
                ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg,
                    struct uof_uword_fixup *, base,
                    page->ucp_neigh_reg_tab, lim);

                if (lim < page->ucp_code_area + sizeof(struct uof_code_area))
                        return EINVAL;

                uca = (struct uof_code_area *)(base + page->ucp_code_area);
                qup->qup_num_micro_words = uca->uca_num_micro_words;

                ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks,
                    struct qat_uof_uword_block *, base,
                    uca->uca_uword_block_tab, lim);

                for (i = 0; i < qup->qup_num_uw_blocks; i++) {
                        u_int uwordoff = ((struct uof_uword_block *)(
                            &qup->qup_uw_blocks[i]))->uub_uword_offset;

                        if (lim < uwordoff)
                                return EINVAL;

                        qup->qup_uw_blocks[i].quub_micro_words =
                            (base + uwordoff);
                }
        }

#undef ASSIGN_OBJ_TAB

        return 0;
}

static int
qat_aefw_uof_parse_images(struct qat_softc *sc)
{
        struct uof_chunk_hdr *uch = NULL;
        int i, error;

        for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) {
                uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch);
                if (uch == NULL)
                        break;

                if (i >= nitems(sc->sc_aefw_uof.qafu_imgs))
                        return ENOENT;

                error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch);
                if (error)
                        return error;

                sc->sc_aefw_uof.qafu_num_imgs++;
        }

        return 0;
}

static int
qat_aefw_uof_parse(struct qat_softc *sc)
{
        struct uof_file_hdr *ufh;
        struct uof_file_chunk_hdr *ufch;
        struct uof_obj_hdr *uoh;
        struct uof_chunk_hdr *uch;
        void *uof = NULL;
        size_t size, uof_size, hdr_size;
        uintptr_t base;
        u_int csum;
        int i;

        size = sc->sc_fw_uof_size;
        if (size < MIN_UOF_SIZE)
                return EINVAL;
        size -= sizeof(struct uof_file_hdr);

        ufh = sc->sc_fw_uof;

        if (ufh->ufh_id != UOF_FID)
                return EINVAL;
        if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER)
                return EINVAL;

        if (ufh->ufh_max_chunks < ufh->ufh_num_chunks)
                return EINVAL;
        if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks)
                return EINVAL;
        ufch = (struct uof_file_chunk_hdr *)(ufh + 1);

        uof_size = 0;
        for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) {
                if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size)
                        return EINVAL;

                if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) {
                        if (uof != NULL)
                                return EINVAL;

                        uof =
                            (void *)((uintptr_t)sc->sc_fw_uof +
                            ufch->ufch_offset);
                        uof_size = ufch->ufch_size;

                        csum = qat_aefw_csum(uof, uof_size);
                        if (csum != ufch->ufch_csum)
                                return EINVAL;
                }
        }

        if (uof == NULL)
                return ENOENT;

        size = uof_size;
        if (size < sizeof(struct uof_obj_hdr))
                return EINVAL;
        size -= sizeof(struct uof_obj_hdr);

        uoh = uof;

        if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks)
                return EINVAL;

        /* Check if the UOF objects are compatible with the chip */
        if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0)
                return ENOTSUP;

        if (uoh->uoh_min_cpu_ver > sc->sc_rev ||
            uoh->uoh_max_cpu_ver < sc->sc_rev)
                return ENOTSUP;

        sc->sc_aefw_uof.qafu_size = uof_size;
        sc->sc_aefw_uof.qafu_obj_hdr = uoh;

        base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;

        /* map uof string-table */
        uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL);
        if (uch != NULL) {
                hdr_size = offsetof(struct uof_str_tab, ust_strings);
                sc->sc_aefw_uof.qafu_str_tab =
                    (void *)(base + uch->uch_offset + hdr_size);
                sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size;
        }

        /* get ustore mem inits table -- should be only one */
        uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL);
        if (uch != NULL) {
                if (uch->uch_size < sizeof(struct uof_obj_table))
                        return EINVAL;
                sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base +
                    uch->uch_offset))->uot_nentries;
                if (sc->sc_aefw_uof.qafu_num_init_mem) {
                        sc->sc_aefw_uof.qafu_init_mem =
                            (struct uof_init_mem *)(base + uch->uch_offset +
                            sizeof(struct uof_obj_table));
                        sc->sc_aefw_uof.qafu_init_mem_size =
                            uch->uch_size - sizeof(struct uof_obj_table);
                }
        }

        uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL);
        if (uch != NULL) {
                if (uch->uch_size < sizeof(struct uof_obj_table) +
                    sizeof(struct uof_var_mem_seg))
                        return EINVAL;
                sc->sc_aefw_uof.qafu_var_mem_seg =
                    (struct uof_var_mem_seg *)(base + uch->uch_offset +
                    sizeof(struct uof_obj_table));
        }

        return qat_aefw_uof_parse_images(sc);
}

static int
qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi,
    struct suof_chunk_hdr *sch)
{
        struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
        struct simg_ae_mode *ae_mode;
        u_int maj_ver;

        qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset +
            sizeof(struct suof_obj_hdr);
        qsi->qsi_simg_len =
            ((struct suof_obj_hdr *)
            (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length;

        qsi->qsi_css_header = qsi->qsi_simg_buf;
        qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr);
        qsi->qsi_css_signature = qsi->qsi_css_key +
            CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;
        qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN;

        ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg;
        qsi->qsi_ae_mask = ae_mode->sam_ae_mask;
        qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name;
        qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data;
        qsi->qsi_fw_type = ae_mode->sam_fw_type;

        if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type)
                return EINVAL;

        maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff;
        if ((maj_ver > ae_mode->sam_devmax_ver) ||
            (maj_ver < ae_mode->sam_devmin_ver)) {
                return EINVAL;
        }

        return 0;
}

static int
qat_aefw_suof_parse(struct qat_softc *sc)
{
        struct suof_file_hdr *sfh;
        struct suof_chunk_hdr *sch;
        struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
        struct qat_suof_image *qsi;
        size_t size;
        u_int csum;
        int ae0_img = MAX_AE;
        int i, error;

        size = sc->sc_fw_suof_size;
        if (size < sizeof(struct suof_file_hdr))
                return EINVAL;

        sfh = sc->sc_fw_suof;

        if (sfh->sfh_file_id != SUOF_FID)
                return EINVAL;
        if (sfh->sfh_fw_type != 0)
                return EINVAL;
        if (sfh->sfh_num_chunks <= 1)
                return EINVAL;
        if (sfh->sfh_min_ver != SUOF_MIN_VER ||
            sfh->sfh_maj_ver != SUOF_MAJ_VER)
                return EINVAL;

        csum = qat_aefw_csum((char *)&sfh->sfh_min_ver,
            size - offsetof(struct suof_file_hdr, sfh_min_ver));
        if (csum != sfh->sfh_check_sum)
                return EINVAL;

        size -= sizeof(struct suof_file_hdr);

        qafs->qafs_file_id = SUOF_FID;
        qafs->qafs_suof_buf = sc->sc_fw_suof;
        qafs->qafs_suof_size = sc->sc_fw_suof_size;
        qafs->qafs_check_sum = sfh->sfh_check_sum;
        qafs->qafs_min_ver = sfh->sfh_min_ver;
        qafs->qafs_maj_ver = sfh->sfh_maj_ver;
        qafs->qafs_fw_type = sfh->sfh_fw_type;

        if (size < sizeof(struct suof_chunk_hdr))
                return EINVAL;
        sch = (struct suof_chunk_hdr *)(sfh + 1);
        size -= sizeof(struct suof_chunk_hdr);

        if (size < sizeof(struct suof_str_tab))
                return EINVAL;
        size -= offsetof(struct suof_str_tab, sst_strings);

        qafs->qafs_sym_size = ((struct suof_str_tab *)
            (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length;
        if (size < qafs->qafs_sym_size)
                return EINVAL;
        qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset +
            offsetof(struct suof_str_tab, sst_strings);

        qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1;
        if (qafs->qafs_num_simgs == 0)
                return EINVAL;

        qsi = qat_alloc_mem(
            sizeof(struct qat_suof_image) * qafs->qafs_num_simgs);
        qafs->qafs_simg = qsi;

        for (i = 0; i < qafs->qafs_num_simgs; i++) {
                error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]);
                if (error)
                        return error;
                if ((qsi[i].qsi_ae_mask & 0x1) != 0)
                        ae0_img = i;
        }

        if (ae0_img != qafs->qafs_num_simgs - 1) {
                struct qat_suof_image last_qsi;

                memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1],
                    sizeof(struct qat_suof_image));
                memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img],
                    sizeof(struct qat_suof_image));
                memcpy(&qsi[ae0_img], &last_qsi,
                    sizeof(struct qat_suof_image));
        }

        return 0;
}

static int
qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size,
    struct qat_dmamem *dma)
{
        struct css_hdr *css = (struct css_hdr *)image;
        struct auth_chunk *auth_chunk;
        struct fw_auth_desc *auth_desc;
        size_t mapsize, simg_offset = sizeof(struct auth_chunk);
        bus_size_t bus_addr;
        uintptr_t virt_addr;
        int error;

        if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN)
                return EINVAL;

        mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ?
            CSS_AE_SIMG_LEN + simg_offset :
            size + CSS_FWSK_PAD_LEN + simg_offset;
        error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE);
        if (error)
                return error;

        memset(dma->qdm_dma_vaddr, 0, mapsize);

        auth_chunk = dma->qdm_dma_vaddr;
        auth_chunk->ac_chunk_size = mapsize;
        auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr;

        virt_addr = (uintptr_t)dma->qdm_dma_vaddr;
        virt_addr += simg_offset;
        bus_addr = auth_chunk->ac_chunk_bus_addr;
        bus_addr += simg_offset;

        auth_desc = &auth_chunk->ac_fw_auth_desc;
        auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32;
        auth_desc->fad_css_hdr_low = bus_addr;

        memcpy((void *)virt_addr, image, sizeof(struct css_hdr));
        /* pub key */
        virt_addr += sizeof(struct css_hdr);
        bus_addr += sizeof(struct css_hdr);
        image += sizeof(struct css_hdr);

        auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32;
        auth_desc->fad_fwsk_pub_low = bus_addr;

        memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN);
        memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN);
        memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN),
            image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t));

        virt_addr += CSS_FWSK_PUB_LEN;
        bus_addr += CSS_FWSK_PUB_LEN;
        image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;

        auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32;
        auth_desc->fad_signature_low = bus_addr;

        memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN);

        virt_addr += CSS_SIGNATURE_LEN;
        bus_addr += CSS_SIGNATURE_LEN;
        image += CSS_SIGNATURE_LEN;

        auth_desc->fad_img_high = (uint64_t)bus_addr >> 32;
        auth_desc->fad_img_low = bus_addr;
        auth_desc->fad_img_len = size - AE_IMG_OFFSET;

        memcpy((void *)virt_addr, image, auth_desc->fad_img_len);

        if (css->css_fw_type == CSS_AE_FIRMWARE) {
                auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high;
                auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low;

                bus_addr += sizeof(struct simg_ae_mode);

                auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32;
                auth_desc->fad_img_ae_init_data_low = bus_addr;

                bus_addr += SIMG_AE_INIT_SEQ_LEN;

                auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32;
                auth_desc->fad_img_ae_insts_low = bus_addr;
        } else {
                auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high;
                auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low;
        }

        bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map,
            BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

        return 0;
}

static int
qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma)
{
        bus_addr_t addr;
        uint32_t fcu, sts;
        int retry = 0;

        addr = dma->qdm_dma_seg.ds_addr;
        qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32);
        qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr);
        qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH);

        do {
                DELAY(FW_AUTH_WAIT_PERIOD * 1000);
                fcu = qat_cap_global_read_4(sc, FCU_STATUS);
                sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
                if (sts == FCU_STATUS_STS_VERI_FAIL)
                        goto fail;
                if (fcu & FCU_STATUS_AUTHFWLD &&
                    sts == FCU_STATUS_STS_VERI_DONE) {
                        return 0;
                }
        } while (retry++ < FW_AUTH_MAX_RETRY);

fail:
        device_printf(sc->sc_dev,
           "firmware authentication error: status 0x%08x retry %d\n",
           fcu, retry);
        return EINVAL;
}

static int
qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma)
{
        struct simg_ae_mode *ae_mode;
        uint32_t fcu, sts, loaded;
        u_int mask;
        u_char ae;
        int retry = 0;

        ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr +
            sizeof(struct auth_chunk) + sizeof(struct css_hdr) +
            CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN);

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                if (!(mask & 1))
                        continue;
                if (!((ae_mode->sam_ae_mask >> ae) & 0x1))
                        continue;
                if (qat_ae_is_active(sc, ae)) {
                        device_printf(sc->sc_dev, "AE %d is active\n", ae);
                        return EINVAL;
                }
                qat_cap_global_write_4(sc, FCU_CTRL,
                    FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE));
                do {
                        DELAY(FW_AUTH_WAIT_PERIOD * 1000);
                        fcu = qat_cap_global_read_4(sc, FCU_STATUS);
                        sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
                        loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE);
                        if (sts == FCU_STATUS_STS_LOAD_DONE &&
                            (loaded & (1 << ae))) {
                                break;
                        }
                } while (retry++ < FW_AUTH_MAX_RETRY);

                if (retry > FW_AUTH_MAX_RETRY) {
                        device_printf(sc->sc_dev,
                            "firmware load timeout: status %08x\n", fcu);
                        return EINVAL;
                }
        }

        return 0;
}

static int
qat_aefw_suof_write(struct qat_softc *sc)
{
        struct qat_suof_image *qsi;
        int i, error = 0;

        for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) {
                qsi = &sc->sc_aefw_suof.qafs_simg[i];
                error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf,
                    qsi->qsi_simg_len, &qsi->qsi_dma);
                if (error)
                        return error;
                error = qat_aefw_auth(sc, &qsi->qsi_dma);
                if (error) {
                        qat_free_dmamem(sc, &qsi->qsi_dma);
                        return error;
                }
                error = qat_aefw_suof_load(sc, &qsi->qsi_dma);
                if (error) {
                        qat_free_dmamem(sc, &qsi->qsi_dma);
                        return error;
                }
                qat_free_dmamem(sc, &qsi->qsi_dma);
        }
        qat_free_mem(sc->sc_aefw_suof.qafs_simg);

        return 0;
}

static int
qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae,
        struct qat_uof_image *qui)
{
        struct qat_ae_slice *slice;
        int i, npages, nregions;

        if (qae->qae_num_slices >= nitems(qae->qae_slices))
                return ENOENT;

        if (qui->qui_image->ui_ae_mode &
            (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) {
                /* XXX */
                device_printf(sc->sc_dev,
                    "shared ae mode is not supported yet\n");
                return ENOTSUP;
        }

        qae->qae_shareable_ustore = 0; /* XXX */
        qae->qae_effect_ustore_size = USTORE_SIZE;

        slice = &qae->qae_slices[qae->qae_num_slices];

        slice->qas_image = qui;
        slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned;

        nregions = qui->qui_image->ui_num_page_regions;
        npages = qui->qui_image->ui_num_pages;

        if (nregions > nitems(slice->qas_regions))
                return ENOENT;
        if (npages > nitems(slice->qas_pages))
                return ENOENT;

        for (i = 0; i < nregions; i++) {
                STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages);
        }
        for (i = 0; i < npages; i++) {
                struct qat_ae_page *page = &slice->qas_pages[i];
                int region;

                page->qap_page = &qui->qui_pages[i];
                region = page->qap_page->qup_page_region;
                if (region >= nregions)
                        return EINVAL;

                page->qap_region = &slice->qas_regions[region];
        }

        qae->qae_num_slices++;

        return 0;
}

static int
qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae)
{
        struct uof_image *image;
        struct qat_ae *qae = &(QAT_AE(sc, ae));
        int s;
        u_char nn_mode;

        for (s = 0; s < qae->qae_num_slices; s++) {
                if (qae->qae_slices[s].qas_image == NULL)
                        continue;

                image = qae->qae_slices[s].qas_image->qui_image;
                qat_ae_write_ctx_mode(sc, ae,
                    __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE));

                nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE);
                if (nn_mode != AE_MODE_NN_MODE_DONTCARE)
                        qat_ae_write_nn_mode(sc, ae, nn_mode);

                qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0,
                    __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0));
                qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1,
                    __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1));

                qat_ae_write_shared_cs_mode(sc, ae,
                    __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE));
                qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size,
                    __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED),
                    qae->qae_reloc_ustore_dram);
        }

        return 0;
}

static int
qat_aefw_uof_init(struct qat_softc *sc)
{
        int ae, i, error;
        uint32_t mask;

        for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
                struct qat_ae *qae;

                if (!(mask & 1))
                        continue;

                qae = &(QAT_AE(sc, ae));

                for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
                        if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned &
                            (1 << ae)) == 0)
                                continue;

                        error = qat_aefw_uof_assign_image(sc, qae,
                            &sc->sc_aefw_uof.qafu_imgs[i]);
                        if (error)
                                return error;
                }

                /* XXX UcLo_initNumUwordUsed */

                qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */

                error = qat_aefw_uof_init_ae(sc, ae);
                if (error)
                        return error;
        }

        return 0;
}

int
qat_aefw_load(struct qat_softc *sc)
{
        int error;

        error = qat_aefw_load_mof(sc);
        if (error)
                return error;

        error = qat_aefw_load_mmp(sc);
        if (error)
                return error;

        error = qat_aefw_mof_parse(sc);
        if (error) {
                device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error);
                return error;
        }

        if (sc->sc_hw.qhw_fw_auth) {
                error = qat_aefw_suof_parse(sc);
                if (error) {
                        device_printf(sc->sc_dev, "couldn't parse suof: %d\n",
                            error);
                        return error;
                }

                error = qat_aefw_suof_write(sc);
                if (error) {
                        device_printf(sc->sc_dev,
                            "could not write firmware: %d\n", error);
                        return error;
                }

        } else {
                error = qat_aefw_uof_parse(sc);
                if (error) {
                        device_printf(sc->sc_dev, "couldn't parse uof: %d\n",
                            error);
                        return error;
                }

                error = qat_aefw_uof_init(sc);
                if (error) {
                        device_printf(sc->sc_dev,
                            "couldn't init for aefw: %d\n", error);
                        return error;
                }

                error = qat_aefw_uof_write(sc);
                if (error) {
                        device_printf(sc->sc_dev,
                            "Could not write firmware: %d\n", error);
                        return error;
                }
        }

        return 0;
}

void
qat_aefw_unload(struct qat_softc *sc)
{
        qat_aefw_unload_mmp(sc);
        qat_aefw_unload_mof(sc);
}

int
qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
        uint32_t fcu;
        int retry = 0;

        if (sc->sc_hw.qhw_fw_auth) {
                qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START);
                do {
                        DELAY(FW_AUTH_WAIT_PERIOD * 1000);
                        fcu = qat_cap_global_read_4(sc, FCU_STATUS);
                        if (fcu & FCU_STATUS_DONE)
                                return 0;
                } while (retry++ < FW_AUTH_MAX_RETRY);

                device_printf(sc->sc_dev,
                    "firmware start timeout: status %08x\n", fcu);
                return EINVAL;
        } else {
                qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX,
                    CTX_WAKEUP_EVENTS_INDIRECT,
                    CTX_WAKEUP_EVENTS_INDIRECT_SLEEP);
                qat_ae_enable_ctx(sc, ae, ctx_mask);
        }

        return 0;
}

static int
qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim)
{
        struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
        struct qat_ae_batch_init_list *qabi_list;
        struct uof_mem_val_attr *memattr;
        size_t *curinit;
        u_long ael;
        int i;
        const char *sym;
        char *ep;

        memattr = (struct uof_mem_val_attr *)(uim + 1);

        switch (uim->uim_region) {
        case LMEM_REGION:
                if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) {
                        device_printf(sc->sc_dev,
                            "Invalid lmem addr or bytes\n");
                        return ENOBUFS;
                }
                if (uim->uim_scope != UOF_SCOPE_LOCAL)
                        return EINVAL;
                sym = qat_aefw_uof_string(sc, uim->uim_sym_name);
                ael = strtoul(sym, &ep, 10);
                if (ep == sym || ael > MAX_AE)
                        return EINVAL;
                if ((sc->sc_ae_mask & (1 << ael)) == 0)
                        return 0; /* ae is fused out */

                curinit = &qafu->qafu_num_lm_init[ael];
                qabi_list = &qafu->qafu_lm_init[ael];

                for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) {
                        struct qat_ae_batch_init *qabi;

                        qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init));
                        if (*curinit == 0)
                                STAILQ_INIT(qabi_list);
                        STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next);

                        qabi->qabi_ae = (u_int)ael;
                        qabi->qabi_addr =
                            uim->uim_addr + memattr->umva_byte_offset;
                        qabi->qabi_value = &memattr->umva_value;
                        qabi->qabi_size = 4;
                        qafu->qafu_num_lm_init_inst[ael] +=
                            qat_ae_get_inst_num(qabi->qabi_size);
                        (*curinit)++;
                        if (*curinit >= MAX_LMEM_REG) {
                                device_printf(sc->sc_dev,
                                    "Invalid lmem val attr\n");
                                return ENOBUFS;
                        }
                }
                break;
        case SRAM_REGION:
        case DRAM_REGION:
        case DRAM1_REGION:
        case SCRATCH_REGION:
        case UMEM_REGION:
                /* XXX */
                /* fallthrough */
        default:
                device_printf(sc->sc_dev,
                    "unsupported memory region to init: %d\n",
                    uim->uim_region);
                return ENOTSUP;
        }

        return 0;
}

static void
qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae)
{
        struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
        struct qat_ae_batch_init *qabi;

        while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) {
                STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next);
                qat_free_mem(qabi);
        }

        qafu->qafu_num_lm_init[ae] = 0;
        qafu->qafu_num_lm_init_inst[ae] = 0;
}

static int
qat_aefw_init_ustore(struct qat_softc *sc)
{
        uint64_t *fill;
        uint32_t dont_init;
        int a, i, p;
        int error = 0;
        int usz, end, start;
        u_char ae, nae;

        fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t));

        for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) {
                struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a];
                struct uof_image *ui = qui->qui_image;

                for (i = 0; i < MAX_USTORE; i++)
                        memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t));
                /*
                 * Compute do_not_init value as a value that will not be equal
                 * to fill data when cast to an int
                 */
                dont_init = 0;
                if (dont_init == (uint32_t)fill[0])
                        dont_init = 0xffffffff;

                for (p = 0; p < ui->ui_num_pages; p++) {
                        struct qat_uof_page *qup = &qui->qui_pages[p];
                        if (!qup->qup_def_page)
                                continue;

                        for (i = qup->qup_beg_paddr;
                            i < qup->qup_beg_paddr + qup->qup_num_micro_words;
                            i++ ) {
                                fill[i] = (uint64_t)dont_init;
                        }
                }

                for (ae = 0; ae < sc->sc_ae_num; ae++) {
                        MPASS(ae < UOF_MAX_NUM_OF_AE);
                        if ((ui->ui_ae_assigned & (1 << ae)) == 0)
                                continue;

                        if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) {
                                qat_ae_get_shared_ustore_ae(ae, &nae);
                                if (ui->ui_ae_assigned & (1 << ae))
                                        continue;
                        }
                        usz = QAT_AE(sc, ae).qae_effect_ustore_size;

                        /* initialize the areas not going to be overwritten */
                        end = -1;
                        do {
                                /* find next uword that needs to be initialized */
                                for (start = end + 1; start < usz; start++) {
                                        if ((uint32_t)fill[start] != dont_init)
                                                break;
                                }
                                /* see if there are no more such uwords */
                                if (start >= usz)
                                        break;
                                for (end = start + 1; end < usz; end++) {
                                        if ((uint32_t)fill[end] == dont_init)
                                                break;
                                }
                                if (QAT_AE(sc, ae).qae_shareable_ustore) {
                                        error = ENOTSUP; /* XXX */
                                        goto out;
                                } else {
                                        error = qat_ae_ucode_write(sc, ae,
                                            start, end - start, &fill[start]);
                                        if (error) {
                                                goto out;
                                        }
                                }

                        } while (end < usz);
                }
        }

out:
        qat_free_mem(fill);
        return error;
}

static int
qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask,
    enum aereg_type regtype, u_short regaddr, u_int value)
{
        int error = 0;
        u_char ctx;

        switch (regtype) {
        case AEREG_GPA_REL:
        case AEREG_GPB_REL:
        case AEREG_SR_REL:
        case AEREG_SR_RD_REL:
        case AEREG_SR_WR_REL:
        case AEREG_DR_REL:
        case AEREG_DR_RD_REL:
        case AEREG_DR_WR_REL:
        case AEREG_NEIGH_REL:
                /* init for all valid ctx */
                for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
                        if ((ctx_mask & (1 << ctx)) == 0)
                                continue;
                        error = qat_aereg_rel_data_write(sc, ae, ctx, regtype,
                            regaddr, value);
                }
                break;
        case AEREG_GPA_ABS:
        case AEREG_GPB_ABS:
        case AEREG_SR_ABS:
        case AEREG_SR_RD_ABS:
        case AEREG_SR_WR_ABS:
        case AEREG_DR_ABS:
        case AEREG_DR_RD_ABS:
        case AEREG_DR_WR_ABS:
                error = qat_aereg_abs_data_write(sc, ae, regtype,
                    regaddr, value);
                break;
        default:
                error = EINVAL;
                break;
        }

        return error;
}

static int
qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae,
    struct qat_uof_image *qui)
{
        u_int i, expres;
        u_char ctx_mask;

        for (i = 0; i < qui->qui_num_init_reg_sym; i++) {
                struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i];

                if (uirs->uirs_value_type == EXPR_VAL) {
                        /* XXX */
                        device_printf(sc->sc_dev,
                            "does not support initializing EXPR_VAL\n");
                        return ENOTSUP;
                } else {
                        expres = uirs->uirs_value;
                }

                switch (uirs->uirs_init_type) {
                case INIT_REG:
                        if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
                            AE_MODE_CTX_MODE) == MAX_AE_CTX) {
                                ctx_mask = 0xff; /* 8-ctx mode */
                        } else {
                                ctx_mask = 0x55; /* 4-ctx mode */
                        }
                        qat_aefw_init_reg(sc, ae, ctx_mask,
                            (enum aereg_type)uirs->uirs_reg_type,
                            (u_short)uirs->uirs_addr_offset, expres);
                        break;
                case INIT_REG_CTX:
                        if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
                            AE_MODE_CTX_MODE) == MAX_AE_CTX) {
                                ctx_mask = 0xff; /* 8-ctx mode */
                        } else {
                                ctx_mask = 0x55; /* 4-ctx mode */
                        }
                        if (((1 << uirs->uirs_ctx) & ctx_mask) == 0)
                                return EINVAL;
                        qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx,
                            (enum aereg_type)uirs->uirs_reg_type,
                            (u_short)uirs->uirs_addr_offset, expres);
                        break;
                case INIT_EXPR:
                case INIT_EXPR_ENDIAN_SWAP:
                default:
                        device_printf(sc->sc_dev,
                            "does not support initializing init_type %d\n",
                            uirs->uirs_init_type);
                        return ENOTSUP;
                }
        }

        return 0;
}

static int
qat_aefw_init_memory(struct qat_softc *sc)
{
        struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
        size_t uimsz, initmemsz = qafu->qafu_init_mem_size;
        struct uof_init_mem *uim;
        int error, i;
        u_char ae;

        uim = qafu->qafu_init_mem;
        for (i = 0; i < qafu->qafu_num_init_mem; i++) {
                uimsz = sizeof(struct uof_init_mem) +
                    sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr;
                if (uimsz > initmemsz) {
                        device_printf(sc->sc_dev,
                            "invalid uof_init_mem or uof_mem_val_attr size\n");
                        return EINVAL;
                }

                if (uim->uim_num_bytes > 0) {
                        error = qat_aefw_init_memory_one(sc, uim);
                        if (error) {
                                device_printf(sc->sc_dev,
                                    "Could not init ae memory: %d\n", error);
                                return error;
                        }
                }
                uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz);
                initmemsz -= uimsz;
        }

        /* run Batch put LM API */
        for (ae = 0; ae < MAX_AE; ae++) {
                error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae],
                    qafu->qafu_num_lm_init_inst[ae]);
                if (error)
                        device_printf(sc->sc_dev, "Could not put lm\n");

                qat_aefw_free_lm_init(sc, ae);
        }

        error = qat_aefw_init_ustore(sc);

        /* XXX run Batch put LM API */

        return error;
}

static int
qat_aefw_init_globals(struct qat_softc *sc)
{
        struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
        int error, i, p, s;
        u_char ae;

        /* initialize the memory segments */
        if (qafu->qafu_num_init_mem > 0) {
                error = qat_aefw_init_memory(sc);
                if (error)
                        return error;
        } else {
                error = qat_aefw_init_ustore(sc);
                if (error)
                        return error;
        }

        /* XXX bind import variables with ivd values */

        /* XXX bind the uC global variables
         * local variables will done on-the-fly */
        for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
                for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) {
                        struct qat_uof_page *qup =
                            &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p];
                        if (qup->qup_num_uw_blocks &&
                            (qup->qup_num_uc_var || qup->qup_num_imp_var)) {
                                device_printf(sc->sc_dev,
                                    "not support uC global variables\n");
                                return ENOTSUP;
                        }
                }
        }

        for (ae = 0; ae < sc->sc_ae_num; ae++) {
                struct qat_ae *qae = &(QAT_AE(sc, ae));

                for (s = 0; s < qae->qae_num_slices; s++) {
                        struct qat_ae_slice *qas = &qae->qae_slices[s];

                        if (qas->qas_image == NULL)
                                continue;

                        error =
                            qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image);
                        if (error)
                                return error;
                }
        }

        return 0;
}

static uint64_t
qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup,
    u_int addr)
{
        uint64_t uinst = 0;
        u_int i;

        /* find the block */
        for (i = 0; i < qup->qup_num_uw_blocks; i++) {
                struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i];

                if ((addr >= quub->quub_start_addr) &&
                    (addr <= (quub->quub_start_addr +
                    (quub->quub_num_words - 1)))) {
                        /* unpack n bytes and assigned to the 64-bit uword value.
                        note: the microwords are stored as packed bytes.
                        */
                        addr -= quub->quub_start_addr;
                        addr *= AEV2_PACKED_UWORD_BYTES;
                        memcpy(&uinst,
                            (void *)((uintptr_t)quub->quub_micro_words + addr),
                            AEV2_PACKED_UWORD_BYTES);
                        uinst = uinst & UWORD_MASK;

                        return uinst;
                }
        }

        return INVLD_UWORD;
}

static int
qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup)
{
        struct qat_ae *qae = &(QAT_AE(sc, ae));
        uint64_t fill, *ucode_cpybuf;
        u_int error, i, upaddr, ninst, cpylen;

        if (qup->qup_num_uc_var || qup->qup_num_neigh_reg ||
            qup->qup_num_imp_var || qup->qup_num_imp_expr) {
                device_printf(sc->sc_dev,
                    "does not support fixup locals\n");
                return ENOTSUP;
        }

        ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t));

        /* XXX get fill-pattern from an image -- they are all the same */
        memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern,
            sizeof(uint64_t));

        upaddr = qup->qup_beg_paddr;
        ninst = qup->qup_num_micro_words;
        while (ninst > 0) {
                cpylen = min(ninst, UWORD_CPYBUF_SIZE);

                /* load the buffer */
                for (i = 0; i < cpylen; i++) {
                        /* keep below code structure in case there are
                         * different handling for shared secnarios */
                        if (!qae->qae_shareable_ustore) {
                                /* qat_aefw_get_uof_inst() takes an address that
                                 * is relative to the start of the page.
                                 * So we don't need to add in the physical
                                 * offset of the page. */
                                if (qup->qup_page_region != 0) {
                                        /* XXX */
                                        device_printf(sc->sc_dev,
                                            "region != 0 is not supported\n");
                                        qat_free_mem(ucode_cpybuf);
                                        return ENOTSUP;
                                } else {
                                        /* for mixing case, it should take
                                         * physical address */
                                        ucode_cpybuf[i] = qat_aefw_get_uof_inst(
                                            sc, qup, upaddr + i);
                                        if (ucode_cpybuf[i] == INVLD_UWORD) {
                                            /* fill hole in the uof */
                                            ucode_cpybuf[i] = fill;
                                        }
                                }
                        } else {
                                /* XXX */
                                qat_free_mem(ucode_cpybuf);
                                return ENOTSUP;
                        }
                }

                /* copy the buffer to ustore */
                if (!qae->qae_shareable_ustore) {
                        error = qat_ae_ucode_write(sc, ae, upaddr, cpylen,
                            ucode_cpybuf);
                        if (error)
                                return error;
                } else {
                        /* XXX */
                        qat_free_mem(ucode_cpybuf);
                        return ENOTSUP;
                }
                upaddr += cpylen;
                ninst -= cpylen;
        }

        qat_free_mem(ucode_cpybuf);

        return 0;
}

static int
qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui)
{
        struct uof_image *ui = qui->qui_image;
        struct qat_ae_page *qap;
        u_int s, p, c;
        int error;
        u_char ae, ctx_mask;

        if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX)
                ctx_mask = 0xff; /* 8-ctx mode */
        else
                ctx_mask = 0x55; /* 4-ctx mode */

        /* load the default page and set assigned CTX PC
         * to the entrypoint address */
        for (ae = 0; ae < sc->sc_ae_num; ae++) {
                struct qat_ae *qae = &(QAT_AE(sc, ae));
                struct qat_ae_slice *qas;
                u_int metadata;

                MPASS(ae < UOF_MAX_NUM_OF_AE);

                if ((ui->ui_ae_assigned & (1 << ae)) == 0)
                        continue;

                /* find the slice to which this image is assigned */
                for (s = 0; s < qae->qae_num_slices; s++) {
                        qas = &qae->qae_slices[s];
                        if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask)
                                break;
                }
                if (s >= qae->qae_num_slices)
                        continue;

                qas = &qae->qae_slices[s];

                for (p = 0; p < ui->ui_num_pages; p++) {
                        qap = &qas->qas_pages[p];

                        /* Only load pages loaded by default */
                        if (!qap->qap_page->qup_def_page)
                                continue;

                        error = qat_aefw_do_pagein(sc, ae, qap->qap_page);
                        if (error)
                                return error;
                }

                metadata = qas->qas_image->qui_image->ui_app_metadata;
                if (metadata != 0xffffffff && bootverbose) {
                        device_printf(sc->sc_dev,
                            "loaded firmware: %s\n",
                            qat_aefw_uof_string(sc, metadata));
                }

                /* Assume starting page is page 0 */
                qap = &qas->qas_pages[0];
                for (c = 0; c < MAX_AE_CTX; c++) {
                        if (ctx_mask & (1 << c))
                                qas->qas_cur_pages[c] = qap;
                        else
                                qas->qas_cur_pages[c] = NULL;
                }

                /* set the live context */
                qae->qae_live_ctx_mask = ui->ui_ctx_assigned;

                /* set context PC to the image entrypoint address */
                error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned,
                    ui->ui_entry_address);
                if (error)
                        return error;
        }

        /* XXX store the checksum for convenience */

        return 0;
}

static int
qat_aefw_uof_write(struct qat_softc *sc)
{
        int error = 0;
        int i;

        error = qat_aefw_init_globals(sc);
        if (error) {
                device_printf(sc->sc_dev,
                    "Could not initialize globals\n");
                return error;
        }

        for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
                error = qat_aefw_uof_write_one(sc,
                    &sc->sc_aefw_uof.qafu_imgs[i]);
                if (error)
                        break;
        }

        /* XXX UcLo_computeFreeUstore */

        return error;
}