/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2024 Arm Ltd
 * Copyright (c) 2022 The FreeBSD Foundation
 *
 * Portions of this software were developed by Andrew Turner under sponsorship
 * from the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * Arm Statistical Profiling Extension (SPE) backend
 *
 * Basic SPE operation
 *
 *   SPE is enabled and configured on a per-core basis, with each core requiring
 *   separate code to enable and configure. Each core also requires a separate
 *   buffer passed as config where the CPU will write profiling data. When the
 *   profiling buffer is full, an interrupt will be taken on the same CPU.
 *
 * Driver Design
 *
 * - HWT allocates a large single buffer per core. This buffer is split in half
 *   to create a 2 element circular buffer (aka ping-pong buffer) where the
 *   kernel writes to one half while userspace is copying the other half
 * - SMP calls are used to enable and configure each core, with SPE initially
 *   configured to write to the first half of the buffer
 * - When the first half of the buffer is full, a buffer full interrupt will
 *   immediately switch writing to the second half. The kernel adds the details
 *   of the half that needs copying to a FIFO STAILQ and notifies userspace via
 *   kqueue by sending a ARM_SPE_KQ_BUF kevent with how many buffers on the
 *   queue need servicing
 * - The kernel responds to HWT_IOC_BUFPTR_GET ioctl by sending details of the
 *   first item from the queue
 * - The buffers pending copying will not be overwritten until an
 *   HWT_IOC_SVC_BUF ioctl is received from userspace confirming the data has
 *   been copied out
 * - In the case where both halfs of the buffer are full, profiling will be
 *   paused until notification via HWT_IOC_SVC_BUF is received
 *
 * Future improvements and limitations
 *
 * - Using large buffer sizes should minimise pauses and loss of profiling
 *   data while kernel is waiting for userspace to copy out data. Since it is
 *   generally expected that consuming (copying) this data is faster than
 *   producing it, in practice this has not so far been an issue. If it does
 *   prove to be an issue even with large buffer sizes then additional buffering
 *   i.e. n element circular buffers might be required.
 *
 * - kqueue can only notify and queue one kevent of the same type, with
 *   subsequent events overwriting data in the first event. The kevent
 *   ARM_SPE_KQ_BUF can therefore only contain the number of buffers on the
 *   STAILQ, incrementing each time a new buffer is full. In this case kqueue
 *   serves just as a notification to userspace to wake up and query the kernel
 *   with the appropriate ioctl. An alternative might be custom kevents where
 *   the kevent identifier is encoded with something like n+cpu_id or n+tid. In
 *   this case data could be sent directly with kqueue via the kevent data and
 *   fflags elements, avoiding the extra ioctl.
 *
 */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/hwt.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mman.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/rman.h>
#include <sys/rwlock.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>

#include <machine/bus.h>

#include <arm64/spe/arm_spe_dev.h>

#include <dev/hwt/hwt_vm.h>
#include <dev/hwt/hwt_backend.h>
#include <dev/hwt/hwt_config.h>
#include <dev/hwt/hwt_context.h>
#include <dev/hwt/hwt_cpu.h>
#include <dev/hwt/hwt_thread.h>

MALLOC_DECLARE(M_ARM_SPE);

extern u_int mp_maxid;
extern struct taskqueue *taskqueue_arm_spe;

int spe_backend_disable_smp(struct hwt_context *ctx);

static device_t spe_dev;
static struct hwt_backend_ops spe_ops;
static struct hwt_backend backend = {
        .ops = &spe_ops,
        .name = "spe",
        .kva_req = 1,
};

/* Pointers to current info structure per CPU. This points to either a per-CPU
 * structure (for CPU mode) or a per-thread structure (for thread mode).
 */
static struct arm_spe_info **spe_info;

static struct arm_spe_info *spe_info_cpu;

static void
spe_backend_init_cpu(struct hwt_context *ctx)
{
        struct arm_spe_info *info;
        struct arm_spe_softc *sc = device_get_softc(spe_dev);
        char lock_name[32];
        char *tmp = "Arm SPE lock/cpu/";
        int cpu_id;

        spe_info_cpu = malloc(sizeof(struct arm_spe_info) * mp_ncpus,
           M_ARM_SPE, M_WAITOK | M_ZERO);


        CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
                info = &spe_info_cpu[cpu_id];
                info->sc = sc;
                info->ident = cpu_id;
                info->buf_info[0].info = info;
                info->buf_info[0].buf_idx = 0;
                info->buf_info[1].info = info;
                info->buf_info[1].buf_idx = 1;
                snprintf(lock_name, sizeof(lock_name), "%s%d", tmp, cpu_id);
                mtx_init(&info->lock, lock_name, NULL, MTX_SPIN);

                spe_info[cpu_id] = info;
        }
}

static int
spe_backend_init(struct hwt_context *ctx)
{
        struct arm_spe_softc *sc = device_get_softc(spe_dev);
        int error = 0;

        /*
         * HWT currently specifies buffer size must be a multiple of PAGE_SIZE,
         * i.e. minimum 4KB + the maximum PMBIDR.Align is 2KB
         * This should never happen but it's good to sense check
         */
        if (ctx->bufsize % sc->kva_align != 0)
                return (EINVAL);

        /*
         * Since we're splitting the buffer in half + PMBLIMITR needs to be page
         * aligned, minimum buffer size needs to be 2x PAGE_SIZE
         */
        if (ctx->bufsize < (2 * PAGE_SIZE))
                return (EINVAL);

        sc->ctx = ctx;
        sc->kqueue_fd = ctx->kqueue_fd;
        sc->hwt_td = ctx->hwt_td;

        spe_info = malloc(sizeof(struct arm_spe_info *) * mp_ncpus,
           M_ARM_SPE, M_WAITOK | M_ZERO);
        sc->spe_info = spe_info;

        if (ctx->mode == HWT_MODE_CPU)
                spe_backend_init_cpu(ctx);

        return (error);
}

#ifdef ARM_SPE_DEBUG
static void hex_dump(uint8_t *buf, size_t len)
{
        size_t i;

        printf("--------------------------------------------------------------\n");
        for (i = 0; i < len; ++i) {
                if (i % 8 == 0) {
                        printf(" ");
                }
                if (i % 16 == 0) {
                        if (i != 0) {
                                printf("\r\n");
                        }
                        printf("\t");
                }
                printf("%02X ", buf[i]);
        }
        printf("\r\n");
}
#endif

static int
spe_backend_deinit(struct hwt_context *ctx)
{
#ifdef ARM_SPE_DEBUG
        struct arm_spe_info *info;
        struct hwt_thread *thr;
        int cpu_id;

        if (ctx->mode == HWT_MODE_CPU) {
                CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
                        info = &spe_info_cpu[cpu_id];
                        printf("CPU %u:\n", cpu_id);
                        hex_dump((void *)info->kvaddr, 128);
                        hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128);
                }
        } else {
                TAILQ_FOREACH(thr, &ctx->threads, next) {
                        info = (struct arm_spe_info *)thr->private;
                        printf("TID %u:\n", thr->thread_id);
                        hex_dump((void *)info->kvaddr, 128);
                        hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128);
                }
        }
#endif

        spe_backend_disable_smp(ctx);

        if (ctx->mode == HWT_MODE_CPU)
                free(spe_info_cpu, M_ARM_SPE);

        free(spe_info, M_ARM_SPE);

        return (0);
}

static uint64_t
arm_spe_min_interval(struct arm_spe_softc *sc)
{
        /* IMPLEMENTATION DEFINED */
        switch (PMSIDR_Interval_VAL(sc->pmsidr))
        {
        case PMSIDR_Interval_256:
                return (256);
        case PMSIDR_Interval_512:
                return (512);
        case PMSIDR_Interval_768:
                return (768);
        case PMSIDR_Interval_1024:
                return (1024);
        case PMSIDR_Interval_1536:
                return (1536);
        case PMSIDR_Interval_2048:
                return (2048);
        case PMSIDR_Interval_3072:
                return (3072);
        case PMSIDR_Interval_4096:
                return (4096);
        default:
                return (4096);
        }
}

static inline void
arm_spe_set_interval(struct arm_spe_info *info, uint64_t interval)
{
        uint64_t min_interval = arm_spe_min_interval(info->sc);

        interval = MAX(interval, min_interval);
        interval = MIN(interval, 1 << 24);      /* max 24 bits */

        dprintf("%s %lu\n", __func__, interval);

        info->pmsirr &= ~(PMSIRR_INTERVAL_MASK);
        info->pmsirr |= (interval << PMSIRR_INTERVAL_SHIFT);
}

static int
spe_backend_configure(struct hwt_context *ctx, int cpu_id, int thread_id)
{
        struct arm_spe_info *info = NULL;
        struct arm_spe_config *cfg;
        struct hwt_thread *thr = NULL;
        int err = 0;

        if (ctx->mode == HWT_MODE_CPU)
                info = &spe_info_cpu[cpu_id];
        else {
                TAILQ_FOREACH(thr, &ctx->threads, next) {
                        if (thr->thread_id != thread_id)
                                continue;
                        info = (struct arm_spe_info *)thr->private;
                        break;
                }
                if (info == NULL)
                        return (ENOENT);
        }

        mtx_lock_spin(&info->lock);
        if (ctx->mode == HWT_MODE_CPU)
                info->ident = cpu_id;
        else
                info->ident = thread_id;
        /* Set defaults */
        info->pmsfcr = 0;
        info->pmsevfr = 0xFFFFFFFFFFFFFFFFUL;
        info->pmslatfr = 0;
        info->pmsirr =
            (arm_spe_min_interval(info->sc) << PMSIRR_INTERVAL_SHIFT)
            | PMSIRR_RND;
        info->pmsicr = 0;
        info->pmscr = PMSCR_TS | PMSCR_PA | PMSCR_CX | PMSCR_E1SPE | PMSCR_E0SPE;

        if (ctx->config != NULL &&
            ctx->config_size == sizeof(struct arm_spe_config) &&
            ctx->config_version == 1) {
                cfg = (struct arm_spe_config *)ctx->config;
                if (cfg->interval)
                        arm_spe_set_interval(info, cfg->interval);
                if (cfg->level == ARM_SPE_KERNEL_ONLY)
                        info->pmscr &= ~(PMSCR_E0SPE); /* turn off user */
                if (cfg->level == ARM_SPE_USER_ONLY)
                        info->pmscr &= ~(PMSCR_E1SPE); /* turn off kern */
                if (cfg->ctx_field)
                        info->ctx_field = cfg->ctx_field;
        } else
                err = (EINVAL);

        if (ctx->mode == HWT_MODE_THREAD) {
                info->kvaddr = thr->vm->kvaddr;
                info->buf_size = ctx->bufsize;
        }

        spe_info[cpu_id] = info;
        mtx_unlock_spin(&info->lock);

        return (err);
}


static void
arm_spe_enable(void *arg __unused)
{
        struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
        struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];
        struct hwt_context *ctx = info->sc->ctx;
        uint64_t base, limit;

        dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));

        mtx_lock_spin(&info->lock);

        if (info->stopped) {
                mtx_unlock_spin(&info->lock);
                return;
        }

        if (info->ctx_field == ARM_SPE_CTX_CPU_ID)
                WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, PCPU_GET(cpuid));

        WRITE_SPECIALREG(PMSFCR_EL1_REG, info->pmsfcr);
        WRITE_SPECIALREG(PMSEVFR_EL1_REG, info->pmsevfr);
        WRITE_SPECIALREG(PMSLATFR_EL1_REG, info->pmslatfr);

        /* Set the sampling interval */
        WRITE_SPECIALREG(PMSIRR_EL1_REG, info->pmsirr);
        isb();

        /* Write 0 here before enabling sampling */
        WRITE_SPECIALREG(PMSICR_EL1_REG, info->pmsicr);
        isb();

        base = buf_start_addr(info->buf_idx, info);
        limit = base + (info->buf_size/2);
        /* Enable the buffer */
        limit &= PMBLIMITR_LIMIT_MASK; /* Zero lower 12 bits */
        limit |= PMBLIMITR_E;
        /* Set the base and limit. Restore base pointer if sampling has previously
         * been enabled for this thread.
         */
        if (buf->pmbptr == 0) {
                WRITE_SPECIALREG(PMBPTR_EL1_REG, base);
        } else {
                WRITE_SPECIALREG(PMBPTR_EL1_REG, buf->pmbptr);
        }
        WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit);
        isb();

        /* Enable sampling */
        WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);
        isb();

        info->enabled = true;

        if (ctx->mode == HWT_MODE_THREAD)
                CPU_SET(PCPU_GET(cpuid), &ctx->cpu_map);

        mtx_unlock_spin(&info->lock);
}

static int
spe_backend_enable_smp(struct hwt_context *ctx)
{
        struct arm_spe_info *info;
        struct hwt_vm *vm;
        int cpu_id;

        KASSERT(ctx->mode == HWT_MODE_CPU, ("%s: should only be called for CPU mode", __func__));

        HWT_CTX_LOCK(ctx);
        CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
                vm = hwt_cpu_get(ctx, cpu_id)->vm;
                KASSERT(spe_info[cpu_id] == &spe_info_cpu[cpu_id], ("%s: spe_info mismatch for cpu_id=%u", __func__, cpu_id));
                info = &spe_info_cpu[cpu_id];

                mtx_lock_spin(&info->lock);
                info->kvaddr = vm->kvaddr;
                info->buf_size = ctx->bufsize;
                mtx_unlock_spin(&info->lock);
        }
        HWT_CTX_UNLOCK(ctx);

        cpu_id = CPU_FFS(&ctx->cpu_map) - 1;
        KASSERT(spe_info[cpu_id] == &spe_info_cpu[cpu_id], ("%s: spe_info mismatch for cpu_id=%u", __func__, cpu_id));
        info = spe_info[cpu_id];
        if (info->ctx_field == ARM_SPE_CTX_PID)
                arm64_pid_in_contextidr = true;
        else
                arm64_pid_in_contextidr = false;

        smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
            arm_spe_enable, smp_no_rendezvous_barrier, NULL);

        return (0);
}

static void
arm_spe_disable_nolock(void)
{
        struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
        struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];
        struct hwt_context *ctx = info->sc->ctx;

        if (!info->enabled)
                return;

        dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));

        /* Disable profiling */
        WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0);
        isb();

        /* Drain any remaining tracing data */
        psb_csync();
        dsb(nsh);

        /* Disable the profiling buffer */
        WRITE_SPECIALREG(PMBLIMITR_EL1_REG, 0);
        isb();

        /* Clear interrupt status reg */
        WRITE_SPECIALREG(PMBSR_EL1_REG, 0x0);

        /* Clear PID/CPU_ID from context ID reg */
        WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, 0);

        buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG);
        info->enabled = false;

        if (ctx->mode == HWT_MODE_THREAD)
                CPU_CLR(PCPU_GET(cpuid), &ctx->cpu_map);
}

void
arm_spe_disable(void *arg __unused)
{
        struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];

        mtx_lock_spin(&info->lock);
        arm_spe_disable_nolock();
        mtx_unlock_spin(&info->lock);
}

int
spe_backend_disable_smp(struct hwt_context *ctx)
{
        struct kevent kev;
        struct arm_spe_info *info;
        struct arm_spe_buf_info *buf;
        int cpu_id;
        int ret;

        if (!CPU_EMPTY(&ctx->cpu_map)) {
                /* Disable and send out remaining data in bufs */
                smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
                    arm_spe_disable, smp_no_rendezvous_barrier, NULL);

                CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
                        info = spe_info[cpu_id];
                        buf = &info->buf_info[info->buf_idx];
                        arm_spe_send_buffer(buf, 0);
                }
        }

        arm64_pid_in_contextidr = false;

        /*
         * Tracing on all CPUs has been disabled, and we've sent write ptr
         * offsets for all bufs - let userspace know it can shutdown
         */
        EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
        ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK);
        if (ret)
                dprintf("%s kqfd_register ret:%d\n", __func__, ret);

        return (0);
}

static void
spe_backend_enable(struct hwt_context *ctx, int cpu_id)
{
        struct arm_spe_info *info;

        if (ctx->mode == HWT_MODE_CPU)
                return;
        KASSERT(curcpu == cpu_id, ("%s: attempting to enable SPE on another cpu", __func__));

        info = spe_info[cpu_id];

        KASSERT(info != NULL, ("%s: info=NULL", __func__));

        if (info->ctx_field == ARM_SPE_CTX_PID)
                arm64_pid_in_contextidr = true;
        else
                arm64_pid_in_contextidr = false;

        arm_spe_enable(NULL);
}

static void
spe_backend_disable(struct hwt_context *ctx, int cpu_id)
{
        struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];

        if (ctx->mode == HWT_MODE_CPU)
                return;

        KASSERT(curcpu == cpu_id, ("%s: attempting to disable SPE on another cpu", __func__));

        mtx_lock_spin(&info->lock);

        if (!info->stopped)
                arm_spe_disable_nolock();

        mtx_unlock_spin(&info->lock);
}

static void
arm_spe_flush(void *arg, int pending __unused)
{
        struct arm_spe_info *info = arg;
        struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];

        arm_spe_send_buffer(buf, 0);
}

static void
spe_backend_stop(struct hwt_context *ctx)
{
        struct arm_spe_info *info;
        struct hwt_thread *thr;

        HWT_CTX_LOCK(ctx);

        if (ctx->mode == HWT_MODE_THREAD) {
                ctx->state = CTX_STATE_STOPPED;

                TAILQ_FOREACH(thr, &ctx->threads, next) {
                        info = (struct arm_spe_info *)thr->private;

                        mtx_lock_spin(&info->lock);

                        info->stopped = true;

                        if (!info->enabled) {
                                /* Not currently tracing. Enqueue buffer for sending */
                                TASK_INIT(&info->flush_task, 0, (task_fn_t *)arm_spe_flush, info);
                                taskqueue_enqueue(taskqueue_arm_spe, &info->flush_task);
                        }
                        /* Otherwise tracing currently active. As this thread has been
                         * marked as stopped, buffer will be sent on next disable
                         */

                        mtx_unlock_spin(&info->lock);
                }

        }

        HWT_CTX_UNLOCK(ctx);

        taskqueue_drain_all(taskqueue_arm_spe);

        spe_backend_disable_smp(ctx);
}

static void
arm_spe_reenable(void *arg __unused)
{
        struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];

        WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);
        isb();
}

static int
spe_backend_svc_buf(struct hwt_context *ctx, void *data, size_t data_size,
    int data_version)
{
        struct arm_spe_info *info = NULL;
        struct arm_spe_buf_info *buf;
        struct arm_spe_svc_buf *s;
        struct hwt_thread *thr;
        int err = 0;
        cpuset_t cpu_set;

        if (data_size != sizeof(struct arm_spe_svc_buf))
                return (E2BIG);

        if (data_version != 1)
                return (EINVAL);

        s = (struct arm_spe_svc_buf *)data;
        if (s->buf_idx > 1)
                return (ENODEV);

        if (ctx->mode == HWT_MODE_CPU) {
                if (s->ident >= mp_ncpus)
                        return (EINVAL);

                info = spe_info[s->ident];
        } else {
                TAILQ_FOREACH(thr, &ctx->threads, next) {
                        if (thr->thread_id != s->ident)
                                continue;
                        info = (struct arm_spe_info *)thr->private;
                        break;
                }

                if (info == NULL)
                        return (ENOENT);
        }

        mtx_lock_spin(&info->lock);

        buf = &info->buf_info[s->buf_idx];

        if (!info->enabled && ctx->mode == HWT_MODE_CPU) {
                err = ENXIO;
                goto end;
        }

        /* Clear the flag the signals buffer needs servicing */
        buf->buf_svc = false;

        /* Re-enable profiling if we've been waiting for this notification */
        if (buf->buf_wait && !info->stopped) {
                CPU_SETOF(s->ident, &cpu_set);

                mtx_unlock_spin(&info->lock);
                smp_rendezvous_cpus(cpu_set, smp_no_rendezvous_barrier,
                    arm_spe_reenable, smp_no_rendezvous_barrier, NULL);
                mtx_lock_spin(&info->lock);

                buf->buf_wait = false;
        }

end:
        mtx_unlock_spin(&info->lock);
        return (err);
}

static int
spe_backend_read(struct hwt_vm *vm, int *ident, vm_offset_t *offset,
    uint64_t *data)
{
        struct arm_spe_queue *q;
        struct arm_spe_softc *sc = device_get_softc(spe_dev);
        int error = 0;

        mtx_lock_spin(&sc->sc_lock);

        /* Return the first pending buffer that needs servicing */
        q = STAILQ_FIRST(&sc->pending);
        if (q == NULL) {
                error = ENOENT;
                goto error;
        }
        *ident = q->ident;
        *offset = q->offset;
        *data = (q->buf_idx << KQ_BUF_POS_SHIFT) |
            (q->partial_rec << KQ_PARTREC_SHIFT) |
            (q->final_buf << KQ_FINAL_BUF_SHIFT);

        STAILQ_REMOVE_HEAD(&sc->pending, next);
        sc->npending--;

error:
        mtx_unlock_spin(&sc->sc_lock);
        if (error)
                return (error);

        free(q, M_ARM_SPE);
        return (0);
}

static int
spe_backend_thread_alloc(struct hwt_thread *thr)
{
        struct arm_spe_softc *sc = device_get_softc(spe_dev);
        char lock_name[32];
        struct arm_spe_info *info;

        info = malloc(sizeof(*info), M_ARM_SPE, M_WAITOK | M_ZERO);

        info->sc = sc;
        info->buf_info[0].info = info;
        info->buf_info[0].buf_idx = 0;
        info->buf_info[1].info = info;
        info->buf_info[1].buf_idx = 1;
        snprintf(lock_name, sizeof(lock_name), "Arm SPE lock/thr/%d", thr->thread_id);
        mtx_init(&info->lock, lock_name, NULL, MTX_SPIN);

        thr->private = info;

        return (0);
}

static void
spe_backend_thread_free(struct hwt_thread *thr)
{
        struct arm_spe_info *info;

        info = (struct arm_spe_info *)thr->private;

        free(info, M_ARM_SPE);
}

static struct hwt_backend_ops spe_ops = {
        .hwt_backend_init = spe_backend_init,
        .hwt_backend_deinit = spe_backend_deinit,

        .hwt_backend_configure = spe_backend_configure,
        .hwt_backend_svc_buf = spe_backend_svc_buf,
        .hwt_backend_stop = spe_backend_stop,

        .hwt_backend_enable = spe_backend_enable,
        .hwt_backend_disable = spe_backend_disable,

        .hwt_backend_enable_smp = spe_backend_enable_smp,
        .hwt_backend_disable_smp = spe_backend_disable_smp,

        .hwt_backend_read = spe_backend_read,

        .hwt_backend_thread_alloc = spe_backend_thread_alloc,
        .hwt_backend_thread_free = spe_backend_thread_free,
};

int
spe_register(device_t dev)
{
        spe_dev = dev;

        return (hwt_backend_register(&backend));
}