/*      $NetBSD: sig_fpu.c,v 1.3 2026/07/15 02:08:18 riastradh Exp $    */

/*
 * Copyright (c) 2026 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__RCSID("$NetBSD: sig_fpu.c,v 1.3 2026/07/15 02:08:18 riastradh Exp $");

#include "sig_fpu.h"

#include <sys/types.h>

#include <x86/cpu_extended_state.h>

#include <cpuid.h>
#include <inttypes.h>
#include <sched.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>

#include "h_macros.h"

#ifdef __x86_64__
#  define       NXMMREGS        16
#  define       NYMMREGS        16
#  define       NZMMREGS        32
#else  /* 32-bit */
#  define       NXMMREGS        8
#  define       NYMMREGS        8
#  define       NZMMREGS        8
#endif

#define X87_CW_RC       __BITS(11,10) /* rounding control (mode) */
#define X87_CW_PC       __BITS(9,8)   /* precision control */
#define X87_CW_SW_EXC   __BITS(5,0)   /* exception bits */

struct cpuid {
        uint32_t edx;
        uint32_t ecx;
        uint32_t ebx;
        uint32_t eax;
};

static struct cpuid
cpuid(uint32_t eax_fn, uint32_t ecx_idx)
{
        struct cpuid c;

        if (!__get_cpuid_count(eax_fn, ecx_idx,
                &c.eax, &c.ebx, &c.ecx, &c.edx))
                memset(&c, 0, sizeof(c));

        return c;
}

__printflike(4,5)
__noinline
static void
hexdump(FILE *fp, const void *buf, size_t len, const char *title, ...)
{
        va_list va;
        const uint8_t *p = buf;
        size_t i;

        fprintf(fp, "# ");
        va_start(va, title);
        vfprintf(fp, title, va);
        va_end(va);
        fprintf(fp, "\n");
        for (i = 0; i < len; i++) {
                if ((i % 8) == 0)
                        fprintf(fp, " ");
                fprintf(fp, " %02hhx", p[i]);
                if ((i % 16) == 15)
                        fprintf(fp, "\n");
        }
        if (i % 16)
                fprintf(fp, "\n");
}

bool
x87_supported(void)
{
#ifdef __x86_64__
        return true;
#else  /* 32-bit */
        return cpuid(0x00000001, 0).edx & __BIT(0);
#endif
}

int
test_x87(volatile bool *ready, const volatile bool *done)
{
        struct save87 s, s1;
        uint32_t rcpc_exc;
        int error = 0;

        /*
         * Gather the current FPU state so we have some reasonable
         * content for the weird stuff when we restore from it.
         */
        asm("wait\n"
            "   fnsave  %0"
            : /*out*/ "=m"(s));

        /*
         * Randomize the floating-point stack entries ST(0),...,ST(7).
         */
        arc4random_buf(s.s87_ac, sizeof(s.s87_ac));

        /*
         * Randomize rounding control, precision control, and a set of
         * exception bits.  We don't want to raise any unmasked
         * exceptions, so we'll raise _and_ mask them.
         *
         * XXX Consider testing the condition code, exception status,
         * and top of stack bits in the status word too.
         */
        rcpc_exc = arc4random() & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC);
        if (__SHIFTOUT(rcpc_exc, X87_CW_PC) == 1) /* reserved */
                rcpc_exc |= __SHIFTIN(3, X87_CW_PC); /* extended */
        s.s87_cw &= ~(X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC);
        s.s87_cw |= rcpc_exc;
        s.s87_sw &= ~X87_CW_SW_EXC;
        s.s87_sw |= rcpc_exc & X87_CW_SW_EXC;

        /*
         * Load up the registers, report that we're ready, busy-wait
         * with the registers still live -- so no subroutine calls --
         * until we're done, and then store the registers back to
         * memory so we can check them.
         */
        asm("\n"
            "   frstor  %[s]\n"
            "   lock\n"
            "   orb     $1,%[ready]\n"
            "0: pause\n"
            "   cmpb    $0,%[done]\n"
            "   lfence\n"
            "   je      0b\n"
            "   wait\n"
            "   fnsave  %[s1]\n"
            : /*out*/ [ready]"=m"(*ready), [s1]"=m"(s1)
            : /*in*/ [done]"m"(*done), [s]"m"(s));

        /*
         * If there are any mismatches between the before and after
         * ST(i) registers, or the control word, or the status word,
         * print them and fail.
         */
        if (memcmp(s.s87_ac, s1.s87_ac, sizeof(s.s87_ac)) != 0) {
                unsigned i;

                for (i = 0; i < 8; i++) {
                        if (memcmp(&s.s87_ac[i], &s1.s87_ac[i],
                                sizeof(s.s87_ac[i])) == 0)
                                continue;
                        fprintf(stderr, "ST(%u)=0x%08x%016"PRIx64","
                            " expected 0x%08x%016"PRIx64"\n",
                            i,
                            s1.s87_ac[i].f87_exp_sign,
                            s1.s87_ac[i].f87_mantissa,
                            s.s87_ac[i].f87_exp_sign,
                            s.s87_ac[i].f87_mantissa);
                        error |= 1 << i;
                }
        }
        if ((s1.s87_cw & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC)) !=
            (s.s87_cw & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC))) {
                fprintf(stderr, "cw=0x%04x, expected 0x%04x,"
                    " diff 0x%04x\n",
                    s1.s87_cw, s.s87_cw,
                    (uint32_t)((s1.s87_cw ^ s.s87_cw) &
                        (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC)));
                error |= 1 << 8;
        }
        if ((s1.s87_sw & X87_CW_SW_EXC) !=
            (s.s87_cw & X87_CW_SW_EXC)) {
                fprintf(stderr, "sw=0x%04x, expected 0x%04x,"
                    " diff 0x%04x\n",
                    s1.s87_sw, s.s87_sw,
                    (uint32_t)((s1.s87_sw ^ s.s87_sw) &
                        X87_CW_SW_EXC));
                error |= 1 << 9;
        }

        return error;
}

void
trash_x87(void)
{
        struct save87 s;
        uint32_t rcpc_exc;

        /*
         * Gather the current FPU state so we have some reasonable
         * content for the weird stuff when we restore from it.
         */
        asm("wait\n"
            "   fnsave  %0"
            : /*out*/ "=m"(s));

        /*
         * Randomize the floating-point stack entries ST(0),...,ST(7).
         */
        arc4random_buf(s.s87_ac, sizeof(s.s87_ac));

        /*
         * Randomize rounding control, precision control, and a set of
         * exception bits.  We don't want to raise any unmasked
         * exceptions, so we'll raise _and_ mask them.
         *
         * XXX Consider trashing the condition code, exception status,
         * and top of stack bits in the status word too.
         */
        rcpc_exc = arc4random() & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC);
        s.s87_cw &= ~(X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC);
        s.s87_cw |= rcpc_exc;
        s.s87_sw &= ~X87_CW_SW_EXC;
        s.s87_sw |= rcpc_exc & X87_CW_SW_EXC;

        /*
         * Load the state into the FPU.
         */
        asm volatile("frstor %0" :: /*in*/ "m"(s));
}

#define MXCSR_MISALIGNEDEXC     __BIT(17)       /* misaligned exc fault */
#define MXCSR_FTZ               __BIT(15)       /* flush to zero */
#define MXCSR_RC                __BITS(14,13)   /* rounding control */
#define MXCSR_EXCMASK           __BITS(12,7)    /* exceptions masked */
#define MXCSR_DAZ               __BIT(6)        /* denormals are zero */
#define MXCSR_EXCFLAG           __BITS(5,0)     /* exceptions raised */

struct xmmregs {
        struct {
                uint8_t b[16];
        } __aligned(16) xmm[NXMMREGS];
        uint32_t mxcsr;
};

bool
xmm_supported(void)
{
#ifdef __x86_64__
        return true;
#else  /* 32-bit */
        return cpuid(0x00000001, 0).edx & __BIT(25);
#endif
}

__attribute__((target("sse")))
int
test_xmm(volatile bool *ready, const volatile bool *done)
{
        struct xmmregs before, after;
        uint32_t exc;
        unsigned i;
        int error = 0;

        /*
         * Randomize the xmm register content.  Randomize plausible
         * bits for the mxcsr.
         */
        arc4random_buf(before.xmm, sizeof(before.xmm));
        before.mxcsr = arc4random() & (MXCSR_FTZ | MXCSR_RC | MXCSR_DAZ);
        exc = arc4random() & 0x3f;
        before.mxcsr |= __SHIFTIN(exc, MXCSR_EXCMASK);
        before.mxcsr |= __SHIFTIN(exc, MXCSR_EXCFLAG);

        /*
         * Load up the registers, report that we're ready, busy-wait
         * with the registers still live -- so no subroutine calls --
         * until we're done, and then store the registers back to
         * memory so we can check them.
         */
        asm("\n"
            "   ldmxcsr %[mxcsr_before]\n"
            "   movdqa  0*16(%[xmm_before]),%%xmm0\n"
            "   movdqa  1*16(%[xmm_before]),%%xmm1\n"
            "   movdqa  2*16(%[xmm_before]),%%xmm2\n"
            "   movdqa  3*16(%[xmm_before]),%%xmm3\n"
            "   movdqa  4*16(%[xmm_before]),%%xmm4\n"
            "   movdqa  5*16(%[xmm_before]),%%xmm5\n"
            "   movdqa  6*16(%[xmm_before]),%%xmm6\n"
            "   movdqa  7*16(%[xmm_before]),%%xmm7\n"
#ifdef __x86_64__
            "   movdqa  8*16(%[xmm_before]),%%xmm8\n"
            "   movdqa  9*16(%[xmm_before]),%%xmm9\n"
            "   movdqa  10*16(%[xmm_before]),%%xmm10\n"
            "   movdqa  11*16(%[xmm_before]),%%xmm11\n"
            "   movdqa  12*16(%[xmm_before]),%%xmm12\n"
            "   movdqa  13*16(%[xmm_before]),%%xmm13\n"
            "   movdqa  14*16(%[xmm_before]),%%xmm14\n"
            "   movdqa  15*16(%[xmm_before]),%%xmm15\n"
#endif
            "   lock\n"
            "   orb     $1,%[ready]\n"
            "0: pause\n"
            "   cmpb    $0,%[done]\n"
            "   lfence\n"
            "   je      0b\n"
            "   movdqa  %%xmm0,0*16(%[xmm_after])\n"
            "   movdqa  %%xmm1,1*16(%[xmm_after])\n"
            "   movdqa  %%xmm2,2*16(%[xmm_after])\n"
            "   movdqa  %%xmm3,3*16(%[xmm_after])\n"
            "   movdqa  %%xmm4,4*16(%[xmm_after])\n"
            "   movdqa  %%xmm5,5*16(%[xmm_after])\n"
            "   movdqa  %%xmm6,6*16(%[xmm_after])\n"
            "   movdqa  %%xmm7,7*16(%[xmm_after])\n"
#ifdef __x86_64__
            "   movdqa  %%xmm8,8*16(%[xmm_after])\n"
            "   movdqa  %%xmm9,9*16(%[xmm_after])\n"
            "   movdqa  %%xmm10,10*16(%[xmm_after])\n"
            "   movdqa  %%xmm11,11*16(%[xmm_after])\n"
            "   movdqa  %%xmm12,12*16(%[xmm_after])\n"
            "   movdqa  %%xmm13,13*16(%[xmm_after])\n"
            "   movdqa  %%xmm14,14*16(%[xmm_after])\n"
            "   movdqa  %%xmm15,15*16(%[xmm_after])\n"
#endif
            "   stmxcsr %[mxcsr_after]\n"
            : /*out*/ [ready]"=m"(*ready), "=m"(after.xmm),
              [mxcsr_after]"=m"(after.mxcsr)
            : /*in*/ [done]"m"(*done), "m"(before.xmm),
              [mxcsr_before]"m"(before.mxcsr),
              [xmm_before]"r"(&before.xmm), [xmm_after]"r"(&after.xmm)
            : /*clobber*/ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
              "xmm6", "xmm7"
#ifdef __x86_64__
            , "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
              "xmm13", "xmm14", "xmm15"
#endif
        );

        /*
         * If there are any mismatches between the before and after xmm
         * registers, or the mxcsr, print them and fail.
         */
        for (i = 0; i < __arraycount(before.xmm); i++) {
                if (memcmp(&before.xmm[i], &after.xmm[i],
                        sizeof(before.xmm[i]))) {
                        fprintf(stderr, "xmm%u clobbered\n", i);
                        hexdump(stderr, &before.xmm[i], sizeof(before.xmm[i]),
                            "before");
                        hexdump(stderr, &after.xmm[i], sizeof(after.xmm[i]),
                            "after");
                        error |= 1 << i;
                }
        }

        if (before.mxcsr != after.mxcsr) {
                fprintf(stderr, "mxcsr clobbered:"
                    " before=0x%08"PRIx32", after=0x%08"PRIx32"\n",
                    before.mxcsr, after.mxcsr);
                error |= 1 << 16;
        }

        return error;
}

__attribute__((target("sse")))
void
trash_xmm(void)
{
        struct xmmregs regs;

        arc4random_buf(&regs, sizeof(regs));

        asm("\n"
            "   movdqa  0*16(%[xmm]),%%xmm0\n"
            "   movdqa  1*16(%[xmm]),%%xmm1\n"
            "   movdqa  2*16(%[xmm]),%%xmm2\n"
            "   movdqa  3*16(%[xmm]),%%xmm3\n"
            "   movdqa  4*16(%[xmm]),%%xmm4\n"
            "   movdqa  5*16(%[xmm]),%%xmm5\n"
            "   movdqa  6*16(%[xmm]),%%xmm6\n"
            "   movdqa  7*16(%[xmm]),%%xmm7\n"
#ifdef __x86_64__
            "   movdqa  8*16(%[xmm]),%%xmm8\n"
            "   movdqa  9*16(%[xmm]),%%xmm9\n"
            "   movdqa  10*16(%[xmm]),%%xmm10\n"
            "   movdqa  11*16(%[xmm]),%%xmm11\n"
            "   movdqa  12*16(%[xmm]),%%xmm12\n"
            "   movdqa  13*16(%[xmm]),%%xmm13\n"
            "   movdqa  14*16(%[xmm]),%%xmm14\n"
            "   movdqa  15*16(%[xmm]),%%xmm15\n"
#endif
            : /*out*/
            : /*in*/ [xmm]"r"(regs.xmm), "m"(regs.xmm)
            : /*clobber*/ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
              "xmm6", "xmm7"
#ifdef __x86_64__
            , "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
              "xmm13", "xmm14", "xmm15"
#endif
        );
}

bool
ymm_supported(void)
{
        return cpuid(0x00000001, 0).ecx & __BIT(28);
}

struct ymmregs {
        struct {
                uint8_t b[32];
        } __aligned(32) ymm[NYMMREGS];
};

__attribute__((target("avx")))
int
test_ymm(volatile bool *ready, const volatile bool *done)
{
        struct ymmregs before, after;
        unsigned i;
        int error = 0;

        /*
         * Randomize the ymm register content.
         */
        arc4random_buf(&before, sizeof(before));

        /*
         * Load up the registers, report that we're ready, busy-wait
         * with the registers still live -- so no subroutine calls --
         * until we're done, and then store the registers back to
         * memory so we can check them.
         */
        asm("\n"
            "   vmovdqa 0*32(%[before_ymm]),%%ymm0\n"
            "   vmovdqa 1*32(%[before_ymm]),%%ymm1\n"
            "   vmovdqa 2*32(%[before_ymm]),%%ymm2\n"
            "   vmovdqa 3*32(%[before_ymm]),%%ymm3\n"
            "   vmovdqa 4*32(%[before_ymm]),%%ymm4\n"
            "   vmovdqa 5*32(%[before_ymm]),%%ymm5\n"
            "   vmovdqa 6*32(%[before_ymm]),%%ymm6\n"
            "   vmovdqa 7*32(%[before_ymm]),%%ymm7\n"
#ifdef __x86_64__
            "   vmovdqa 8*32(%[before_ymm]),%%ymm8\n"
            "   vmovdqa 9*32(%[before_ymm]),%%ymm9\n"
            "   vmovdqa 10*32(%[before_ymm]),%%ymm10\n"
            "   vmovdqa 11*32(%[before_ymm]),%%ymm11\n"
            "   vmovdqa 12*32(%[before_ymm]),%%ymm12\n"
            "   vmovdqa 13*32(%[before_ymm]),%%ymm13\n"
            "   vmovdqa 14*32(%[before_ymm]),%%ymm14\n"
            "   vmovdqa 15*32(%[before_ymm]),%%ymm15\n"
#endif
            "   lock\n"
            "   orb     $1,%[ready]\n"
            "0: pause\n"
            "   cmpb    $0,%[done]\n"
            "   lfence\n"
            "   je      0b\n"
            "   vmovdqa %%ymm0,0*32(%[after_ymm])\n"
            "   vmovdqa %%ymm1,1*32(%[after_ymm])\n"
            "   vmovdqa %%ymm2,2*32(%[after_ymm])\n"
            "   vmovdqa %%ymm3,3*32(%[after_ymm])\n"
            "   vmovdqa %%ymm4,4*32(%[after_ymm])\n"
            "   vmovdqa %%ymm5,5*32(%[after_ymm])\n"
            "   vmovdqa %%ymm6,6*32(%[after_ymm])\n"
            "   vmovdqa %%ymm7,7*32(%[after_ymm])\n"
#ifdef __x86_64__
            "   vmovdqa %%ymm8,8*32(%[after_ymm])\n"
            "   vmovdqa %%ymm9,9*32(%[after_ymm])\n"
            "   vmovdqa %%ymm10,10*32(%[after_ymm])\n"
            "   vmovdqa %%ymm11,11*32(%[after_ymm])\n"
            "   vmovdqa %%ymm12,12*32(%[after_ymm])\n"
            "   vmovdqa %%ymm13,13*32(%[after_ymm])\n"
            "   vmovdqa %%ymm14,14*32(%[after_ymm])\n"
            "   vmovdqa %%ymm15,15*32(%[after_ymm])\n"
#endif
            : /*out*/ [ready]"=m"(*ready), "=m"(after.ymm)
            : /*in*/ [done]"m"(*done), "m"(before.ymm),
              [before_ymm]"r"(before.ymm), [after_ymm]"r"(after.ymm)
            : /*clobber*/ "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5",
              "ymm6", "ymm7"
#ifdef __x86_64__
            , "ymm8", "ymm9", "ymm10", "ymm11", "ymm12",
              "ymm13", "ymm14", "ymm15"
#endif
        );

        for (i = 0; i < __arraycount(before.ymm); i++) {
                if (memcmp(&before.ymm[i], &after.ymm[i],
                        sizeof(before.ymm[i]))) {
                        fprintf(stderr, "ymm%u clobbered\n", i);
                        hexdump(stderr, &before.ymm[i], sizeof(before.ymm[i]),
                            "before");
                        hexdump(stderr, &after.ymm[i], sizeof(after.ymm[i]),
                            "after");
                        error |= 1 << i;
                }
        }

        return error;
}

__attribute__((target("avx")))
void
trash_ymm(void)
{
        struct ymmregs regs;

        arc4random_buf(&regs, sizeof(regs));

        asm("\n"
            "   vmovdqa 0*32(%[ymm]),%%ymm0\n"
            "   vmovdqa 1*32(%[ymm]),%%ymm1\n"
            "   vmovdqa 2*32(%[ymm]),%%ymm2\n"
            "   vmovdqa 3*32(%[ymm]),%%ymm3\n"
            "   vmovdqa 4*32(%[ymm]),%%ymm4\n"
            "   vmovdqa 5*32(%[ymm]),%%ymm5\n"
            "   vmovdqa 6*32(%[ymm]),%%ymm6\n"
            "   vmovdqa 7*32(%[ymm]),%%ymm7\n"
#ifdef __x86_64__
            "   vmovdqa 8*32(%[ymm]),%%ymm8\n"
            "   vmovdqa 9*32(%[ymm]),%%ymm9\n"
            "   vmovdqa 10*32(%[ymm]),%%ymm10\n"
            "   vmovdqa 11*32(%[ymm]),%%ymm11\n"
            "   vmovdqa 12*32(%[ymm]),%%ymm12\n"
            "   vmovdqa 13*32(%[ymm]),%%ymm13\n"
            "   vmovdqa 14*32(%[ymm]),%%ymm14\n"
            "   vmovdqa 15*32(%[ymm]),%%ymm15\n"
#endif
            : /*out*/
            : /*in*/ "m"(regs.ymm), [ymm]"r"(regs.ymm)
            : /*clobber*/ "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5",
              "ymm6", "ymm7"
#ifdef __x86_64__
            , "ymm8", "ymm9", "ymm10", "ymm11", "ymm12",
              "ymm13", "ymm14", "ymm15"
#endif
        );
}

bool
zmm_supported(void)
{
        return cpuid(0x00000007, 0).ebx & __BIT(16);
}

struct zmmregs {
        struct {
                uint8_t b[64];
        } __aligned(64) zmm[NZMMREGS];
};

__attribute__((target("avx512f")))
int
test_zmm(volatile bool *ready, const volatile bool *done)
{
        struct zmmregs before, after;
        unsigned i;
        int error = 0;

        /*
         * Randomize the zmm register content.
         */
        arc4random_buf(&before, sizeof(before));

        /*
         * Load up the registers, report that we're ready, busy-wait
         * with the registers still live -- so no subroutine calls --
         * until we're done, and then store the registers back to
         * memory so we can check them.
         */
        asm("\n"
            "   vmovdqa64       0*64(%[before_zmm]),%%zmm0\n"
            "   vmovdqa64       1*64(%[before_zmm]),%%zmm1\n"
            "   vmovdqa64       2*64(%[before_zmm]),%%zmm2\n"
            "   vmovdqa64       3*64(%[before_zmm]),%%zmm3\n"
            "   vmovdqa64       4*64(%[before_zmm]),%%zmm4\n"
            "   vmovdqa64       5*64(%[before_zmm]),%%zmm5\n"
            "   vmovdqa64       6*64(%[before_zmm]),%%zmm6\n"
            "   vmovdqa64       7*64(%[before_zmm]),%%zmm7\n"
#ifdef __x86_64__
            "   vmovdqa64       8*64(%[before_zmm]),%%zmm8\n"
            "   vmovdqa64       9*64(%[before_zmm]),%%zmm9\n"
            "   vmovdqa64       10*64(%[before_zmm]),%%zmm10\n"
            "   vmovdqa64       11*64(%[before_zmm]),%%zmm11\n"
            "   vmovdqa64       12*64(%[before_zmm]),%%zmm12\n"
            "   vmovdqa64       13*64(%[before_zmm]),%%zmm13\n"
            "   vmovdqa64       14*64(%[before_zmm]),%%zmm14\n"
            "   vmovdqa64       15*64(%[before_zmm]),%%zmm15\n"
            "   vmovdqa64       16*64(%[before_zmm]),%%zmm16\n"
            "   vmovdqa64       17*64(%[before_zmm]),%%zmm17\n"
            "   vmovdqa64       18*64(%[before_zmm]),%%zmm18\n"
            "   vmovdqa64       19*64(%[before_zmm]),%%zmm19\n"
            "   vmovdqa64       20*64(%[before_zmm]),%%zmm20\n"
            "   vmovdqa64       21*64(%[before_zmm]),%%zmm21\n"
            "   vmovdqa64       22*64(%[before_zmm]),%%zmm22\n"
            "   vmovdqa64       23*64(%[before_zmm]),%%zmm23\n"
            "   vmovdqa64       24*64(%[before_zmm]),%%zmm24\n"
            "   vmovdqa64       25*64(%[before_zmm]),%%zmm25\n"
            "   vmovdqa64       26*64(%[before_zmm]),%%zmm26\n"
            "   vmovdqa64       27*64(%[before_zmm]),%%zmm27\n"
            "   vmovdqa64       28*64(%[before_zmm]),%%zmm28\n"
            "   vmovdqa64       29*64(%[before_zmm]),%%zmm29\n"
            "   vmovdqa64       30*64(%[before_zmm]),%%zmm30\n"
            "   vmovdqa64       31*64(%[before_zmm]),%%zmm31\n"
#endif
            "   lock\n"
            "   orb     $1,%[ready]\n"
            "0: pause\n"
            "   cmpb    $0,%[done]\n"
            "   lfence\n"
            "   je      0b\n"
            "   vmovdqa64       %%zmm0,0*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm1,1*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm2,2*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm3,3*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm4,4*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm5,5*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm6,6*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm7,7*64(%[after_zmm])\n"
#ifdef __x86_64__
            "   vmovdqa64       %%zmm8,8*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm9,9*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm10,10*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm11,11*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm12,12*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm13,13*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm14,14*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm15,15*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm16,16*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm17,17*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm18,18*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm19,19*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm20,20*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm21,21*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm22,22*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm23,23*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm24,24*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm25,25*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm26,26*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm27,27*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm28,28*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm29,29*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm30,30*64(%[after_zmm])\n"
            "   vmovdqa64       %%zmm31,31*64(%[after_zmm])\n"
#endif
            : /*out*/ [ready]"=m"(*ready), "=m"(after.zmm)
            : /*in*/ [done]"m"(*done), "m"(before.zmm),
              [before_zmm]"r"(before.zmm), [after_zmm]"r"(after.zmm)
            : /*clobber*/ "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5",
              "zmm6", "zmm7"
#ifdef __x86_64__
            , "zmm8", "zmm9", "zmm10", "zmm11", "zmm12",
              "zmm13", "zmm14", "zmm15", "zmm16", "zmm17",
              "zmm18", "zmm19", "zmm20", "zmm21", "zmm22",
              "zmm23", "zmm24", "zmm25", "zmm26", "zmm27",
              "zmm28", "zmm29", "zmm30", "zmm31"
#endif
        );

        for (i = 0; i < __arraycount(before.zmm); i++) {
                if (memcmp(&before.zmm[i], &after.zmm[i],
                        sizeof(before.zmm[i]))) {
                        fprintf(stderr, "zmm%u clobbered\n", i);
                        hexdump(stderr, &before.zmm[i], sizeof(before.zmm[i]),
                            "before");
                        hexdump(stderr, &after.zmm[i], sizeof(after.zmm[i]),
                            "after");
                        error |= 1 << i;
                }
        }

        return error;
}

__attribute__((target("avx512f")))
void
trash_zmm(void)
{
        struct zmmregs regs;

        arc4random_buf(&regs, sizeof(regs));

        asm("\n"
            "   vmovdqa64       0*64(%[zmm]),%%zmm0\n"
            "   vmovdqa64       1*64(%[zmm]),%%zmm1\n"
            "   vmovdqa64       2*64(%[zmm]),%%zmm2\n"
            "   vmovdqa64       3*64(%[zmm]),%%zmm3\n"
            "   vmovdqa64       4*64(%[zmm]),%%zmm4\n"
            "   vmovdqa64       5*64(%[zmm]),%%zmm5\n"
            "   vmovdqa64       6*64(%[zmm]),%%zmm6\n"
            "   vmovdqa64       7*64(%[zmm]),%%zmm7\n"
#ifdef __x86_64__
            "   vmovdqa64       8*64(%[zmm]),%%zmm8\n"
            "   vmovdqa64       9*64(%[zmm]),%%zmm9\n"
            "   vmovdqa64       10*64(%[zmm]),%%zmm10\n"
            "   vmovdqa64       11*64(%[zmm]),%%zmm11\n"
            "   vmovdqa64       12*64(%[zmm]),%%zmm12\n"
            "   vmovdqa64       13*64(%[zmm]),%%zmm13\n"
            "   vmovdqa64       14*64(%[zmm]),%%zmm14\n"
            "   vmovdqa64       15*64(%[zmm]),%%zmm15\n"
            "   vmovdqa64       16*64(%[zmm]),%%zmm16\n"
            "   vmovdqa64       17*64(%[zmm]),%%zmm17\n"
            "   vmovdqa64       18*64(%[zmm]),%%zmm18\n"
            "   vmovdqa64       19*64(%[zmm]),%%zmm19\n"
            "   vmovdqa64       20*64(%[zmm]),%%zmm20\n"
            "   vmovdqa64       21*64(%[zmm]),%%zmm21\n"
            "   vmovdqa64       22*64(%[zmm]),%%zmm22\n"
            "   vmovdqa64       23*64(%[zmm]),%%zmm23\n"
            "   vmovdqa64       24*64(%[zmm]),%%zmm24\n"
            "   vmovdqa64       25*64(%[zmm]),%%zmm25\n"
            "   vmovdqa64       26*64(%[zmm]),%%zmm26\n"
            "   vmovdqa64       27*64(%[zmm]),%%zmm27\n"
            "   vmovdqa64       28*64(%[zmm]),%%zmm28\n"
            "   vmovdqa64       29*64(%[zmm]),%%zmm29\n"
            "   vmovdqa64       30*64(%[zmm]),%%zmm30\n"
            "   vmovdqa64       31*64(%[zmm]),%%zmm31\n"
#endif
            : /*out*/
            : /*in*/ "m"(regs.zmm), [zmm]"r"(regs.zmm)
            : /*clobber*/ "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5",
              "zmm6", "zmm7"
#ifdef __x86_64__
            , "zmm8", "zmm9", "zmm10", "zmm11", "zmm12",
              "zmm13", "zmm14", "zmm15", "zmm16", "zmm17",
              "zmm18", "zmm19", "zmm20", "zmm21", "zmm22",
              "zmm23", "zmm24", "zmm25", "zmm26", "zmm27",
              "zmm28", "zmm29", "zmm30", "zmm31"
#endif
        );
}