/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2009, Intel Corporation
 * All rights reserved.
 */

/*       Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.        */
/*       Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T          */
/*         All Rights Reserved                                          */

/*       Copyright (c) 1987, 1988 Microsoft Corporation                 */
/*         All Rights Reserved                                          */

/*
 * Copyright 2020 Joyent, Inc.
 */

#include <sys/errno.h>
#include <sys/asm_linkage.h>

#include "assym.h"

#define KCOPY_MIN_SIZE  128     /* Must be >= 16 bytes */
#define XCOPY_MIN_SIZE  128     /* Must be >= 16 bytes */
/*
 * Non-temopral access (NTA) alignment requirement
 */
#define NTA_ALIGN_SIZE  4       /* Must be at least 4-byte aligned */
#define NTA_ALIGN_MASK  _CONST(NTA_ALIGN_SIZE-1)
#define COUNT_ALIGN_SIZE        16      /* Must be at least 16-byte aligned */
#define COUNT_ALIGN_MASK        _CONST(COUNT_ALIGN_SIZE-1)

/*
 * With the introduction of Broadwell, Intel has introduced supervisor mode
 * access protection -- SMAP. SMAP forces the kernel to set certain bits to
 * enable access of user pages (AC in rflags, defines as PS_ACHK in
 * <sys/psw.h>). One of the challenges is that the implementation of many of the
 * userland copy routines directly use the kernel ones. For example, copyin and
 * copyout simply go and jump to the do_copy_fault label and traditionally let
 * those deal with the return for them. In fact, changing that is a can of frame
 * pointers.
 *
 * Rules and Constraints:
 *
 * 1. For anything that's not in copy.s, we have it do explicit smap_disable()
 * or smap_enable() calls.  This is restricted to the following three places:
 * DTrace, resume() in swtch.s and on_fault/no_fault. If you want to add it
 * somewhere else, we should be thinking twice.
 *
 * 2. We try to toggle this at the smallest window possible. This means that if
 * we take a fault, need to try to use a copyop in copyin() or copyout(), or any
 * other function, we will always leave with SMAP enabled (the kernel cannot
 * access user pages).
 *
 * 3. None of the *_noerr() or ucopy/uzero routines should toggle SMAP. They are
 * explicitly only allowed to be called while in an on_fault()/no_fault() handler,
 * which already takes care of ensuring that SMAP is enabled and disabled. Note
 * this means that when under an on_fault()/no_fault() handler, one must not
 * call the non-*_noerr() routines.
 *
 * 4. The first thing we should do after coming out of an lofault handler is to
 * make sure that we call smap_enable() again to ensure that we are safely
 * protected, as more often than not, we will have disabled smap to get there.
 *
 * 5. smap_enable() and smap_disable() don't exist: calls to these functions
 * generate runtime relocations, that are then processed into the necessary
 * clac/stac, via the krtld hotinlines mechanism and hotinline_smap().
 *
 * 6. For any inline user of SMAP, the appropriate SMAP_ENABLE_INSTR and
 * SMAP_DISABLE_INSTR macro should be used. If the number of these is changed,
 * you must update the constants SMAP_ENABLE_COUNT and SMAP_DISABLE_COUNT below.
 *
 * 7. Generally this .s file is processed by a K&R style cpp. This means that it
 * really has a lot of feelings about whitespace. In particular, if you have a
 * macro FOO with the arguments FOO(1, 3), the second argument is in fact ' 3'.
 *
 * 8. In general, the kernel has its own value for rflags that gets used. This
 * is maintained in a few different places which vary based on how the thread
 * comes into existence and whether it's a user thread. In general, when the
 * kernel takes a trap, it always will set ourselves to a known set of flags,
 * mainly as part of ENABLE_INTR_FLAGS and F_OFF and F_ON. These ensure that
 * PS_ACHK is cleared for us. In addition, when using the sysenter instruction,
 * we mask off PS_ACHK off via the AMD_SFMASK MSR. See init_cpu_syscall() for
 * where that gets masked off.
 */

/*
 * The optimal 64-bit bcopy and kcopy for modern x86 processors uses
 * "rep smovq" for large sizes. Performance data shows that many calls to
 * bcopy/kcopy/bzero/kzero operate on small buffers. For best performance for
 * these small sizes unrolled code is used. For medium sizes loops writing
 * 64-bytes per loop are used. Transition points were determined experimentally.
 */
#define BZERO_USE_REP   (1024)
#define BCOPY_DFLT_REP  (128)
#define BCOPY_NHM_REP   (768)

/*
 * Copy a block of storage, returning an error code if `from' or
 * `to' takes a kernel pagefault which cannot be resolved.
 * Returns errno value on pagefault error, 0 if all ok
 */

/*
 * I'm sorry about these macros, but copy.s is unsurprisingly sensitive to
 * additional call instructions.
 */
#define SMAP_DISABLE_COUNT      16
#define SMAP_ENABLE_COUNT       26

#define SMAP_DISABLE_INSTR(ITER)                \
        .globl  _smap_disable_patch_##ITER;     \
        _smap_disable_patch_##ITER##:;  \
        nop; nop; nop;

#define SMAP_ENABLE_INSTR(ITER)                 \
        .globl  _smap_enable_patch_##ITER;      \
        _smap_enable_patch_##ITER##:;   \
        nop; nop; nop;

        .globl  kernelbase
        .globl  postbootkernelbase

        ENTRY(kcopy)
        pushq   %rbp
        movq    %rsp, %rbp
#ifdef DEBUG
        cmpq    postbootkernelbase(%rip), %rdi          /* %rdi = from */
        jb      0f
        cmpq    postbootkernelbase(%rip), %rsi          /* %rsi = to */
        jnb     1f
0:      leaq    .kcopy_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif
        /*
         * pass lofault value as 4th argument to do_copy_fault
         */
        leaq    _kcopy_copyerr(%rip), %rcx
        movq    %gs:CPU_THREAD, %r9     /* %r9 = thread addr */

do_copy_fault:
        movq    T_LOFAULT(%r9), %r11    /* save the current lofault */
        movq    %rcx, T_LOFAULT(%r9)    /* new lofault */
        call    bcopy_altentry
        xorl    %eax, %eax              /* return 0 (success) */
        SMAP_ENABLE_INSTR(0)

        /*
         * A fault during do_copy_fault is indicated through an errno value
         * in %rax and we iretq from the trap handler to here.
         */
_kcopy_copyerr:
        movq    %r11, T_LOFAULT(%r9)    /* restore original lofault */
        leave
        ret
        SET_SIZE(kcopy)

#undef  ARG_FROM
#undef  ARG_TO
#undef  ARG_COUNT

#define COPY_LOOP_INIT(src, dst, cnt)   \
        addq    cnt, src;                       \
        addq    cnt, dst;                       \
        shrq    $3, cnt;                        \
        neg     cnt

        /* Copy 16 bytes per loop.  Uses %rax and %r8 */
#define COPY_LOOP_BODY(src, dst, cnt)   \
        prefetchnta     0x100(src, cnt, 8);     \
        movq    (src, cnt, 8), %rax;            \
        movq    0x8(src, cnt, 8), %r8;          \
        movnti  %rax, (dst, cnt, 8);            \
        movnti  %r8, 0x8(dst, cnt, 8);          \
        addq    $2, cnt

        ENTRY(kcopy_nta)
        pushq   %rbp
        movq    %rsp, %rbp
#ifdef DEBUG
        cmpq    postbootkernelbase(%rip), %rdi          /* %rdi = from */
        jb      0f
        cmpq    postbootkernelbase(%rip), %rsi          /* %rsi = to */
        jnb     1f
0:      leaq    .kcopy_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif

        movq    %gs:CPU_THREAD, %r9
        cmpq    $0, %rcx                /* No non-temporal access? */
        /*
         * pass lofault value as 4th argument to do_copy_fault
         */
        leaq    _kcopy_nta_copyerr(%rip), %rcx  /* doesn't set rflags */
        jnz     do_copy_fault           /* use regular access */
        /*
         * Make sure cnt is >= KCOPY_MIN_SIZE
         */
        cmpq    $KCOPY_MIN_SIZE, %rdx
        jb      do_copy_fault

        /*
         * Make sure src and dst are NTA_ALIGN_SIZE aligned,
         * count is COUNT_ALIGN_SIZE aligned.
         */
        movq    %rdi, %r10
        orq     %rsi, %r10
        andq    $NTA_ALIGN_MASK, %r10
        orq     %rdx, %r10
        andq    $COUNT_ALIGN_MASK, %r10
        jnz     do_copy_fault

        ALTENTRY(do_copy_fault_nta)
        movq    %gs:CPU_THREAD, %r9     /* %r9 = thread addr */
        movq    T_LOFAULT(%r9), %r11    /* save the current lofault */
        movq    %rcx, T_LOFAULT(%r9)    /* new lofault */

        /*
         * COPY_LOOP_BODY uses %rax and %r8
         */
        COPY_LOOP_INIT(%rdi, %rsi, %rdx)
2:      COPY_LOOP_BODY(%rdi, %rsi, %rdx)
        jnz     2b

        mfence
        xorl    %eax, %eax              /* return 0 (success) */
        SMAP_ENABLE_INSTR(1)

_kcopy_nta_copyerr:
        movq    %r11, T_LOFAULT(%r9)    /* restore original lofault */
        leave
        ret
        SET_SIZE(do_copy_fault_nta)
        SET_SIZE(kcopy_nta)

        ENTRY(bcopy)
#ifdef DEBUG
        orq     %rdx, %rdx              /* %rdx = count */
        jz      1f
        cmpq    postbootkernelbase(%rip), %rdi          /* %rdi = from */
        jb      0f
        cmpq    postbootkernelbase(%rip), %rsi          /* %rsi = to */
        jnb     1f
0:      leaq    .bcopy_panic_msg(%rip), %rdi
        jmp     call_panic              /* setup stack and call panic */
1:
#endif
        /*
         * bcopy_altentry() is called from kcopy, i.e., do_copy_fault.
         * kcopy assumes that bcopy doesn't touch %r9 and %r11. If bcopy
         * uses these registers in future they must be saved and restored.
         */
        ALTENTRY(bcopy_altentry)
do_copy:
#define L(s) .bcopy##s
        cmpq    $0x50, %rdx             /* 80 */
        jae     bcopy_ck_size

        /*
         * Performance data shows many caller's copy small buffers. So for
         * best perf for these sizes unrolled code is used. Store data without
         * worrying about alignment.
         */
        leaq    L(fwdPxQx)(%rip), %r10
        addq    %rdx, %rdi
        addq    %rdx, %rsi
        movslq  (%r10,%rdx,4), %rcx
        leaq    (%rcx,%r10,1), %r10
        INDIRECT_JMP_REG(r10)

        .p2align 4
L(fwdPxQx):
        .int       L(P0Q0)-L(fwdPxQx)   /* 0 */
        .int       L(P1Q0)-L(fwdPxQx)
        .int       L(P2Q0)-L(fwdPxQx)
        .int       L(P3Q0)-L(fwdPxQx)
        .int       L(P4Q0)-L(fwdPxQx)
        .int       L(P5Q0)-L(fwdPxQx)
        .int       L(P6Q0)-L(fwdPxQx)
        .int       L(P7Q0)-L(fwdPxQx)

        .int       L(P0Q1)-L(fwdPxQx)   /* 8 */
        .int       L(P1Q1)-L(fwdPxQx)
        .int       L(P2Q1)-L(fwdPxQx)
        .int       L(P3Q1)-L(fwdPxQx)
        .int       L(P4Q1)-L(fwdPxQx)
        .int       L(P5Q1)-L(fwdPxQx)
        .int       L(P6Q1)-L(fwdPxQx)
        .int       L(P7Q1)-L(fwdPxQx)

        .int       L(P0Q2)-L(fwdPxQx)   /* 16 */
        .int       L(P1Q2)-L(fwdPxQx)
        .int       L(P2Q2)-L(fwdPxQx)
        .int       L(P3Q2)-L(fwdPxQx)
        .int       L(P4Q2)-L(fwdPxQx)
        .int       L(P5Q2)-L(fwdPxQx)
        .int       L(P6Q2)-L(fwdPxQx)
        .int       L(P7Q2)-L(fwdPxQx)

        .int       L(P0Q3)-L(fwdPxQx)   /* 24 */
        .int       L(P1Q3)-L(fwdPxQx)
        .int       L(P2Q3)-L(fwdPxQx)
        .int       L(P3Q3)-L(fwdPxQx)
        .int       L(P4Q3)-L(fwdPxQx)
        .int       L(P5Q3)-L(fwdPxQx)
        .int       L(P6Q3)-L(fwdPxQx)
        .int       L(P7Q3)-L(fwdPxQx)

        .int       L(P0Q4)-L(fwdPxQx)   /* 32 */
        .int       L(P1Q4)-L(fwdPxQx)
        .int       L(P2Q4)-L(fwdPxQx)
        .int       L(P3Q4)-L(fwdPxQx)
        .int       L(P4Q4)-L(fwdPxQx)
        .int       L(P5Q4)-L(fwdPxQx)
        .int       L(P6Q4)-L(fwdPxQx)
        .int       L(P7Q4)-L(fwdPxQx)

        .int       L(P0Q5)-L(fwdPxQx)   /* 40 */
        .int       L(P1Q5)-L(fwdPxQx)
        .int       L(P2Q5)-L(fwdPxQx)
        .int       L(P3Q5)-L(fwdPxQx)
        .int       L(P4Q5)-L(fwdPxQx)
        .int       L(P5Q5)-L(fwdPxQx)
        .int       L(P6Q5)-L(fwdPxQx)
        .int       L(P7Q5)-L(fwdPxQx)

        .int       L(P0Q6)-L(fwdPxQx)   /* 48 */
        .int       L(P1Q6)-L(fwdPxQx)
        .int       L(P2Q6)-L(fwdPxQx)
        .int       L(P3Q6)-L(fwdPxQx)
        .int       L(P4Q6)-L(fwdPxQx)
        .int       L(P5Q6)-L(fwdPxQx)
        .int       L(P6Q6)-L(fwdPxQx)
        .int       L(P7Q6)-L(fwdPxQx)

        .int       L(P0Q7)-L(fwdPxQx)   /* 56 */
        .int       L(P1Q7)-L(fwdPxQx)
        .int       L(P2Q7)-L(fwdPxQx)
        .int       L(P3Q7)-L(fwdPxQx)
        .int       L(P4Q7)-L(fwdPxQx)
        .int       L(P5Q7)-L(fwdPxQx)
        .int       L(P6Q7)-L(fwdPxQx)
        .int       L(P7Q7)-L(fwdPxQx)

        .int       L(P0Q8)-L(fwdPxQx)   /* 64 */
        .int       L(P1Q8)-L(fwdPxQx)
        .int       L(P2Q8)-L(fwdPxQx)
        .int       L(P3Q8)-L(fwdPxQx)
        .int       L(P4Q8)-L(fwdPxQx)
        .int       L(P5Q8)-L(fwdPxQx)
        .int       L(P6Q8)-L(fwdPxQx)
        .int       L(P7Q8)-L(fwdPxQx)

        .int       L(P0Q9)-L(fwdPxQx)   /* 72 */
        .int       L(P1Q9)-L(fwdPxQx)
        .int       L(P2Q9)-L(fwdPxQx)
        .int       L(P3Q9)-L(fwdPxQx)
        .int       L(P4Q9)-L(fwdPxQx)
        .int       L(P5Q9)-L(fwdPxQx)
        .int       L(P6Q9)-L(fwdPxQx)
        .int       L(P7Q9)-L(fwdPxQx)   /* 79 */

        .p2align 4
L(P0Q9):
        mov    -0x48(%rdi), %rcx
        mov    %rcx, -0x48(%rsi)
L(P0Q8):
        mov    -0x40(%rdi), %r10
        mov    %r10, -0x40(%rsi)
L(P0Q7):
        mov    -0x38(%rdi), %r8
        mov    %r8, -0x38(%rsi)
L(P0Q6):
        mov    -0x30(%rdi), %rcx
        mov    %rcx, -0x30(%rsi)
L(P0Q5):
        mov    -0x28(%rdi), %r10
        mov    %r10, -0x28(%rsi)
L(P0Q4):
        mov    -0x20(%rdi), %r8
        mov    %r8, -0x20(%rsi)
L(P0Q3):
        mov    -0x18(%rdi), %rcx
        mov    %rcx, -0x18(%rsi)
L(P0Q2):
        mov    -0x10(%rdi), %r10
        mov    %r10, -0x10(%rsi)
L(P0Q1):
        mov    -0x8(%rdi), %r8
        mov    %r8, -0x8(%rsi)
L(P0Q0):
        ret

        .p2align 4
L(P1Q9):
        mov    -0x49(%rdi), %r8
        mov    %r8, -0x49(%rsi)
L(P1Q8):
        mov    -0x41(%rdi), %rcx
        mov    %rcx, -0x41(%rsi)
L(P1Q7):
        mov    -0x39(%rdi), %r10
        mov    %r10, -0x39(%rsi)
L(P1Q6):
        mov    -0x31(%rdi), %r8
        mov    %r8, -0x31(%rsi)
L(P1Q5):
        mov    -0x29(%rdi), %rcx
        mov    %rcx, -0x29(%rsi)
L(P1Q4):
        mov    -0x21(%rdi), %r10
        mov    %r10, -0x21(%rsi)
L(P1Q3):
        mov    -0x19(%rdi), %r8
        mov    %r8, -0x19(%rsi)
L(P1Q2):
        mov    -0x11(%rdi), %rcx
        mov    %rcx, -0x11(%rsi)
L(P1Q1):
        mov    -0x9(%rdi), %r10
        mov    %r10, -0x9(%rsi)
L(P1Q0):
        movzbq -0x1(%rdi), %r8
        mov    %r8b, -0x1(%rsi)
        ret

        .p2align 4
L(P2Q9):
        mov    -0x4a(%rdi), %r8
        mov    %r8, -0x4a(%rsi)
L(P2Q8):
        mov    -0x42(%rdi), %rcx
        mov    %rcx, -0x42(%rsi)
L(P2Q7):
        mov    -0x3a(%rdi), %r10
        mov    %r10, -0x3a(%rsi)
L(P2Q6):
        mov    -0x32(%rdi), %r8
        mov    %r8, -0x32(%rsi)
L(P2Q5):
        mov    -0x2a(%rdi), %rcx
        mov    %rcx, -0x2a(%rsi)
L(P2Q4):
        mov    -0x22(%rdi), %r10
        mov    %r10, -0x22(%rsi)
L(P2Q3):
        mov    -0x1a(%rdi), %r8
        mov    %r8, -0x1a(%rsi)
L(P2Q2):
        mov    -0x12(%rdi), %rcx
        mov    %rcx, -0x12(%rsi)
L(P2Q1):
        mov    -0xa(%rdi), %r10
        mov    %r10, -0xa(%rsi)
L(P2Q0):
        movzwq -0x2(%rdi), %r8
        mov    %r8w, -0x2(%rsi)
        ret

        .p2align 4
L(P3Q9):
        mov    -0x4b(%rdi), %r8
        mov    %r8, -0x4b(%rsi)
L(P3Q8):
        mov    -0x43(%rdi), %rcx
        mov    %rcx, -0x43(%rsi)
L(P3Q7):
        mov    -0x3b(%rdi), %r10
        mov    %r10, -0x3b(%rsi)
L(P3Q6):
        mov    -0x33(%rdi), %r8
        mov    %r8, -0x33(%rsi)
L(P3Q5):
        mov    -0x2b(%rdi), %rcx
        mov    %rcx, -0x2b(%rsi)
L(P3Q4):
        mov    -0x23(%rdi), %r10
        mov    %r10, -0x23(%rsi)
L(P3Q3):
        mov    -0x1b(%rdi), %r8
        mov    %r8, -0x1b(%rsi)
L(P3Q2):
        mov    -0x13(%rdi), %rcx
        mov    %rcx, -0x13(%rsi)
L(P3Q1):
        mov    -0xb(%rdi), %r10
        mov    %r10, -0xb(%rsi)
        /*
         * These trailing loads/stores have to do all their loads 1st,
         * then do the stores.
         */
L(P3Q0):
        movzwq -0x3(%rdi), %r8
        movzbq -0x1(%rdi), %r10
        mov    %r8w, -0x3(%rsi)
        mov    %r10b, -0x1(%rsi)
        ret

        .p2align 4
L(P4Q9):
        mov    -0x4c(%rdi), %r8
        mov    %r8, -0x4c(%rsi)
L(P4Q8):
        mov    -0x44(%rdi), %rcx
        mov    %rcx, -0x44(%rsi)
L(P4Q7):
        mov    -0x3c(%rdi), %r10
        mov    %r10, -0x3c(%rsi)
L(P4Q6):
        mov    -0x34(%rdi), %r8
        mov    %r8, -0x34(%rsi)
L(P4Q5):
        mov    -0x2c(%rdi), %rcx
        mov    %rcx, -0x2c(%rsi)
L(P4Q4):
        mov    -0x24(%rdi), %r10
        mov    %r10, -0x24(%rsi)
L(P4Q3):
        mov    -0x1c(%rdi), %r8
        mov    %r8, -0x1c(%rsi)
L(P4Q2):
        mov    -0x14(%rdi), %rcx
        mov    %rcx, -0x14(%rsi)
L(P4Q1):
        mov    -0xc(%rdi), %r10
        mov    %r10, -0xc(%rsi)
L(P4Q0):
        mov    -0x4(%rdi), %r8d
        mov    %r8d, -0x4(%rsi)
        ret

        .p2align 4
L(P5Q9):
        mov    -0x4d(%rdi), %r8
        mov    %r8, -0x4d(%rsi)
L(P5Q8):
        mov    -0x45(%rdi), %rcx
        mov    %rcx, -0x45(%rsi)
L(P5Q7):
        mov    -0x3d(%rdi), %r10
        mov    %r10, -0x3d(%rsi)
L(P5Q6):
        mov    -0x35(%rdi), %r8
        mov    %r8, -0x35(%rsi)
L(P5Q5):
        mov    -0x2d(%rdi), %rcx
        mov    %rcx, -0x2d(%rsi)
L(P5Q4):
        mov    -0x25(%rdi), %r10
        mov    %r10, -0x25(%rsi)
L(P5Q3):
        mov    -0x1d(%rdi), %r8
        mov    %r8, -0x1d(%rsi)
L(P5Q2):
        mov    -0x15(%rdi), %rcx
        mov    %rcx, -0x15(%rsi)
L(P5Q1):
        mov    -0xd(%rdi), %r10
        mov    %r10, -0xd(%rsi)
L(P5Q0):
        mov    -0x5(%rdi), %r8d
        movzbq -0x1(%rdi), %r10
        mov    %r8d, -0x5(%rsi)
        mov    %r10b, -0x1(%rsi)
        ret

        .p2align 4
L(P6Q9):
        mov    -0x4e(%rdi), %r8
        mov    %r8, -0x4e(%rsi)
L(P6Q8):
        mov    -0x46(%rdi), %rcx
        mov    %rcx, -0x46(%rsi)
L(P6Q7):
        mov    -0x3e(%rdi), %r10
        mov    %r10, -0x3e(%rsi)
L(P6Q6):
        mov    -0x36(%rdi), %r8
        mov    %r8, -0x36(%rsi)
L(P6Q5):
        mov    -0x2e(%rdi), %rcx
        mov    %rcx, -0x2e(%rsi)
L(P6Q4):
        mov    -0x26(%rdi), %r10
        mov    %r10, -0x26(%rsi)
L(P6Q3):
        mov    -0x1e(%rdi), %r8
        mov    %r8, -0x1e(%rsi)
L(P6Q2):
        mov    -0x16(%rdi), %rcx
        mov    %rcx, -0x16(%rsi)
L(P6Q1):
        mov    -0xe(%rdi), %r10
        mov    %r10, -0xe(%rsi)
L(P6Q0):
        mov    -0x6(%rdi), %r8d
        movzwq -0x2(%rdi), %r10
        mov    %r8d, -0x6(%rsi)
        mov    %r10w, -0x2(%rsi)
        ret

        .p2align 4
L(P7Q9):
        mov    -0x4f(%rdi), %r8
        mov    %r8, -0x4f(%rsi)
L(P7Q8):
        mov    -0x47(%rdi), %rcx
        mov    %rcx, -0x47(%rsi)
L(P7Q7):
        mov    -0x3f(%rdi), %r10
        mov    %r10, -0x3f(%rsi)
L(P7Q6):
        mov    -0x37(%rdi), %r8
        mov    %r8, -0x37(%rsi)
L(P7Q5):
        mov    -0x2f(%rdi), %rcx
        mov    %rcx, -0x2f(%rsi)
L(P7Q4):
        mov    -0x27(%rdi), %r10
        mov    %r10, -0x27(%rsi)
L(P7Q3):
        mov    -0x1f(%rdi), %r8
        mov    %r8, -0x1f(%rsi)
L(P7Q2):
        mov    -0x17(%rdi), %rcx
        mov    %rcx, -0x17(%rsi)
L(P7Q1):
        mov    -0xf(%rdi), %r10
        mov    %r10, -0xf(%rsi)
L(P7Q0):
        mov    -0x7(%rdi), %r8d
        movzwq -0x3(%rdi), %r10
        movzbq -0x1(%rdi), %rcx
        mov    %r8d, -0x7(%rsi)
        mov    %r10w, -0x3(%rsi)
        mov    %cl, -0x1(%rsi)
        ret

        /*
         * For large sizes rep smovq is fastest.
         * Transition point determined experimentally as measured on
         * Intel Xeon processors (incl. Nehalem and previous generations) and
         * AMD Opteron. The transition value is patched at boot time to avoid
         * memory reference hit.
         */
        .globl bcopy_patch_start
bcopy_patch_start:
        cmpq    $BCOPY_NHM_REP, %rdx
        .globl bcopy_patch_end
bcopy_patch_end:

        .p2align 4
        ALTENTRY(bcopy_ck_size)

        cmpq    $BCOPY_DFLT_REP, %rdx
        jae     L(use_rep)

        /*
         * Align to a 8-byte boundary. Avoids penalties from unaligned stores
         * as well as from stores spanning cachelines.
         */
        test    $0x7, %rsi
        jz      L(aligned_loop)
        test    $0x1, %rsi
        jz      2f
        movzbq  (%rdi), %r8
        dec     %rdx
        inc     %rdi
        mov     %r8b, (%rsi)
        inc     %rsi
2:
        test    $0x2, %rsi
        jz      4f
        movzwq  (%rdi), %r8
        sub     $0x2, %rdx
        add     $0x2, %rdi
        mov     %r8w, (%rsi)
        add     $0x2, %rsi
4:
        test    $0x4, %rsi
        jz      L(aligned_loop)
        mov     (%rdi), %r8d
        sub     $0x4, %rdx
        add     $0x4, %rdi
        mov     %r8d, (%rsi)
        add     $0x4, %rsi

        /*
         * Copy 64-bytes per loop
         */
        .p2align 4
L(aligned_loop):
        mov     (%rdi), %r8
        mov     0x8(%rdi), %r10
        lea     -0x40(%rdx), %rdx
        mov     %r8, (%rsi)
        mov     %r10, 0x8(%rsi)
        mov     0x10(%rdi), %rcx
        mov     0x18(%rdi), %r8
        mov     %rcx, 0x10(%rsi)
        mov     %r8, 0x18(%rsi)

        cmp     $0x40, %rdx
        mov     0x20(%rdi), %r10
        mov     0x28(%rdi), %rcx
        mov     %r10, 0x20(%rsi)
        mov     %rcx, 0x28(%rsi)
        mov     0x30(%rdi), %r8
        mov     0x38(%rdi), %r10
        lea     0x40(%rdi), %rdi
        mov     %r8, 0x30(%rsi)
        mov     %r10, 0x38(%rsi)
        lea     0x40(%rsi), %rsi
        jae     L(aligned_loop)

        /*
         * Copy remaining bytes (0-63)
         */
L(do_remainder):
        leaq    L(fwdPxQx)(%rip), %r10
        addq    %rdx, %rdi
        addq    %rdx, %rsi
        movslq  (%r10,%rdx,4), %rcx
        leaq    (%rcx,%r10,1), %r10
        INDIRECT_JMP_REG(r10)

        /*
         * Use rep smovq. Clear remainder via unrolled code
         */
        .p2align 4
L(use_rep):
        xchgq   %rdi, %rsi              /* %rsi = source, %rdi = destination */
        movq    %rdx, %rcx              /* %rcx = count */
        shrq    $3, %rcx                /* 8-byte word count */
        rep
          smovq

        xchgq   %rsi, %rdi              /* %rdi = src, %rsi = destination */
        andq    $7, %rdx                /* remainder */
        jnz     L(do_remainder)
        ret
#undef  L
        SET_SIZE(bcopy_ck_size)

#ifdef DEBUG
        /*
         * Setup frame on the run-time stack. The end of the input argument
         * area must be aligned on a 16 byte boundary. The stack pointer %rsp,
         * always points to the end of the latest allocated stack frame.
         * panic(const char *format, ...) is a varargs function. When a
         * function taking variable arguments is called, %rax must be set
         * to eight times the number of floating point parameters passed
         * to the function in SSE registers.
         */
call_panic:
        pushq   %rbp                    /* align stack properly */
        movq    %rsp, %rbp
        xorl    %eax, %eax              /* no variable arguments */
        call    panic                   /* %rdi = format string */
#endif
        SET_SIZE(bcopy_altentry)
        SET_SIZE(bcopy)


/*
 * Zero a block of storage, returning an error code if we
 * take a kernel pagefault which cannot be resolved.
 * Returns errno value on pagefault error, 0 if all ok
 */

        ENTRY(kzero)
#ifdef DEBUG
        cmpq    postbootkernelbase(%rip), %rdi  /* %rdi = addr */
        jnb     0f
        leaq    .kzero_panic_msg(%rip), %rdi
        jmp     call_panic              /* setup stack and call panic */
0:
#endif
        /*
         * pass lofault value as 3rd argument for fault return
         */
        leaq    _kzeroerr(%rip), %rdx

        movq    %gs:CPU_THREAD, %r9     /* %r9 = thread addr */
        movq    T_LOFAULT(%r9), %r11    /* save the current lofault */
        movq    %rdx, T_LOFAULT(%r9)    /* new lofault */
        call    bzero_altentry
        xorl    %eax, %eax
        movq    %r11, T_LOFAULT(%r9)    /* restore the original lofault */
        ret
        /*
         * A fault during bzero is indicated through an errno value
         * in %rax when we iretq to here.
         */
_kzeroerr:
        addq    $8, %rsp                /* pop bzero_altentry call ret addr */
        movq    %r11, T_LOFAULT(%r9)    /* restore the original lofault */
        ret
        SET_SIZE(kzero)

/*
 * Zero a block of storage.
 */

        ENTRY(bzero)
#ifdef DEBUG
        cmpq    postbootkernelbase(%rip), %rdi  /* %rdi = addr */
        jnb     0f
        leaq    .bzero_panic_msg(%rip), %rdi
        jmp     call_panic              /* setup stack and call panic */
0:
#endif
        ALTENTRY(bzero_altentry)
do_zero:
#define L(s) .bzero##s
        xorl    %eax, %eax

        cmpq    $0x50, %rsi             /* 80 */
        jae     L(ck_align)

        /*
         * Performance data shows many caller's are zeroing small buffers. So
         * for best perf for these sizes unrolled code is used. Store zeros
         * without worrying about alignment.
         */
        leaq    L(setPxQx)(%rip), %r10
        addq    %rsi, %rdi
        movslq  (%r10,%rsi,4), %rcx
        leaq    (%rcx,%r10,1), %r10
        INDIRECT_JMP_REG(r10)

        .p2align 4
L(setPxQx):
        .int       L(P0Q0)-L(setPxQx)   /* 0 */
        .int       L(P1Q0)-L(setPxQx)
        .int       L(P2Q0)-L(setPxQx)
        .int       L(P3Q0)-L(setPxQx)
        .int       L(P4Q0)-L(setPxQx)
        .int       L(P5Q0)-L(setPxQx)
        .int       L(P6Q0)-L(setPxQx)
        .int       L(P7Q0)-L(setPxQx)

        .int       L(P0Q1)-L(setPxQx)   /* 8 */
        .int       L(P1Q1)-L(setPxQx)
        .int       L(P2Q1)-L(setPxQx)
        .int       L(P3Q1)-L(setPxQx)
        .int       L(P4Q1)-L(setPxQx)
        .int       L(P5Q1)-L(setPxQx)
        .int       L(P6Q1)-L(setPxQx)
        .int       L(P7Q1)-L(setPxQx)

        .int       L(P0Q2)-L(setPxQx)   /* 16 */
        .int       L(P1Q2)-L(setPxQx)
        .int       L(P2Q2)-L(setPxQx)
        .int       L(P3Q2)-L(setPxQx)
        .int       L(P4Q2)-L(setPxQx)
        .int       L(P5Q2)-L(setPxQx)
        .int       L(P6Q2)-L(setPxQx)
        .int       L(P7Q2)-L(setPxQx)

        .int       L(P0Q3)-L(setPxQx)   /* 24 */
        .int       L(P1Q3)-L(setPxQx)
        .int       L(P2Q3)-L(setPxQx)
        .int       L(P3Q3)-L(setPxQx)
        .int       L(P4Q3)-L(setPxQx)
        .int       L(P5Q3)-L(setPxQx)
        .int       L(P6Q3)-L(setPxQx)
        .int       L(P7Q3)-L(setPxQx)

        .int       L(P0Q4)-L(setPxQx)   /* 32 */
        .int       L(P1Q4)-L(setPxQx)
        .int       L(P2Q4)-L(setPxQx)
        .int       L(P3Q4)-L(setPxQx)
        .int       L(P4Q4)-L(setPxQx)
        .int       L(P5Q4)-L(setPxQx)
        .int       L(P6Q4)-L(setPxQx)
        .int       L(P7Q4)-L(setPxQx)

        .int       L(P0Q5)-L(setPxQx)   /* 40 */
        .int       L(P1Q5)-L(setPxQx)
        .int       L(P2Q5)-L(setPxQx)
        .int       L(P3Q5)-L(setPxQx)
        .int       L(P4Q5)-L(setPxQx)
        .int       L(P5Q5)-L(setPxQx)
        .int       L(P6Q5)-L(setPxQx)
        .int       L(P7Q5)-L(setPxQx)

        .int       L(P0Q6)-L(setPxQx)   /* 48 */
        .int       L(P1Q6)-L(setPxQx)
        .int       L(P2Q6)-L(setPxQx)
        .int       L(P3Q6)-L(setPxQx)
        .int       L(P4Q6)-L(setPxQx)
        .int       L(P5Q6)-L(setPxQx)
        .int       L(P6Q6)-L(setPxQx)
        .int       L(P7Q6)-L(setPxQx)

        .int       L(P0Q7)-L(setPxQx)   /* 56 */
        .int       L(P1Q7)-L(setPxQx)
        .int       L(P2Q7)-L(setPxQx)
        .int       L(P3Q7)-L(setPxQx)
        .int       L(P4Q7)-L(setPxQx)
        .int       L(P5Q7)-L(setPxQx)
        .int       L(P6Q7)-L(setPxQx)
        .int       L(P7Q7)-L(setPxQx)

        .int       L(P0Q8)-L(setPxQx)   /* 64 */
        .int       L(P1Q8)-L(setPxQx)
        .int       L(P2Q8)-L(setPxQx)
        .int       L(P3Q8)-L(setPxQx)
        .int       L(P4Q8)-L(setPxQx)
        .int       L(P5Q8)-L(setPxQx)
        .int       L(P6Q8)-L(setPxQx)
        .int       L(P7Q8)-L(setPxQx)

        .int       L(P0Q9)-L(setPxQx)   /* 72 */
        .int       L(P1Q9)-L(setPxQx)
        .int       L(P2Q9)-L(setPxQx)
        .int       L(P3Q9)-L(setPxQx)
        .int       L(P4Q9)-L(setPxQx)
        .int       L(P5Q9)-L(setPxQx)
        .int       L(P6Q9)-L(setPxQx)
        .int       L(P7Q9)-L(setPxQx)   /* 79 */

        .p2align 4
L(P0Q9): mov    %rax, -0x48(%rdi)
L(P0Q8): mov    %rax, -0x40(%rdi)
L(P0Q7): mov    %rax, -0x38(%rdi)
L(P0Q6): mov    %rax, -0x30(%rdi)
L(P0Q5): mov    %rax, -0x28(%rdi)
L(P0Q4): mov    %rax, -0x20(%rdi)
L(P0Q3): mov    %rax, -0x18(%rdi)
L(P0Q2): mov    %rax, -0x10(%rdi)
L(P0Q1): mov    %rax, -0x8(%rdi)
L(P0Q0):
         ret

        .p2align 4
L(P1Q9): mov    %rax, -0x49(%rdi)
L(P1Q8): mov    %rax, -0x41(%rdi)
L(P1Q7): mov    %rax, -0x39(%rdi)
L(P1Q6): mov    %rax, -0x31(%rdi)
L(P1Q5): mov    %rax, -0x29(%rdi)
L(P1Q4): mov    %rax, -0x21(%rdi)
L(P1Q3): mov    %rax, -0x19(%rdi)
L(P1Q2): mov    %rax, -0x11(%rdi)
L(P1Q1): mov    %rax, -0x9(%rdi)
L(P1Q0): mov    %al, -0x1(%rdi)
         ret

        .p2align 4
L(P2Q9): mov    %rax, -0x4a(%rdi)
L(P2Q8): mov    %rax, -0x42(%rdi)
L(P2Q7): mov    %rax, -0x3a(%rdi)
L(P2Q6): mov    %rax, -0x32(%rdi)
L(P2Q5): mov    %rax, -0x2a(%rdi)
L(P2Q4): mov    %rax, -0x22(%rdi)
L(P2Q3): mov    %rax, -0x1a(%rdi)
L(P2Q2): mov    %rax, -0x12(%rdi)
L(P2Q1): mov    %rax, -0xa(%rdi)
L(P2Q0): mov    %ax, -0x2(%rdi)
         ret

        .p2align 4
L(P3Q9): mov    %rax, -0x4b(%rdi)
L(P3Q8): mov    %rax, -0x43(%rdi)
L(P3Q7): mov    %rax, -0x3b(%rdi)
L(P3Q6): mov    %rax, -0x33(%rdi)
L(P3Q5): mov    %rax, -0x2b(%rdi)
L(P3Q4): mov    %rax, -0x23(%rdi)
L(P3Q3): mov    %rax, -0x1b(%rdi)
L(P3Q2): mov    %rax, -0x13(%rdi)
L(P3Q1): mov    %rax, -0xb(%rdi)
L(P3Q0): mov    %ax, -0x3(%rdi)
         mov    %al, -0x1(%rdi)
         ret

        .p2align 4
L(P4Q9): mov    %rax, -0x4c(%rdi)
L(P4Q8): mov    %rax, -0x44(%rdi)
L(P4Q7): mov    %rax, -0x3c(%rdi)
L(P4Q6): mov    %rax, -0x34(%rdi)
L(P4Q5): mov    %rax, -0x2c(%rdi)
L(P4Q4): mov    %rax, -0x24(%rdi)
L(P4Q3): mov    %rax, -0x1c(%rdi)
L(P4Q2): mov    %rax, -0x14(%rdi)
L(P4Q1): mov    %rax, -0xc(%rdi)
L(P4Q0): mov    %eax, -0x4(%rdi)
         ret

        .p2align 4
L(P5Q9): mov    %rax, -0x4d(%rdi)
L(P5Q8): mov    %rax, -0x45(%rdi)
L(P5Q7): mov    %rax, -0x3d(%rdi)
L(P5Q6): mov    %rax, -0x35(%rdi)
L(P5Q5): mov    %rax, -0x2d(%rdi)
L(P5Q4): mov    %rax, -0x25(%rdi)
L(P5Q3): mov    %rax, -0x1d(%rdi)
L(P5Q2): mov    %rax, -0x15(%rdi)
L(P5Q1): mov    %rax, -0xd(%rdi)
L(P5Q0): mov    %eax, -0x5(%rdi)
         mov    %al, -0x1(%rdi)
         ret

        .p2align 4
L(P6Q9): mov    %rax, -0x4e(%rdi)
L(P6Q8): mov    %rax, -0x46(%rdi)
L(P6Q7): mov    %rax, -0x3e(%rdi)
L(P6Q6): mov    %rax, -0x36(%rdi)
L(P6Q5): mov    %rax, -0x2e(%rdi)
L(P6Q4): mov    %rax, -0x26(%rdi)
L(P6Q3): mov    %rax, -0x1e(%rdi)
L(P6Q2): mov    %rax, -0x16(%rdi)
L(P6Q1): mov    %rax, -0xe(%rdi)
L(P6Q0): mov    %eax, -0x6(%rdi)
         mov    %ax, -0x2(%rdi)
         ret

        .p2align 4
L(P7Q9): mov    %rax, -0x4f(%rdi)
L(P7Q8): mov    %rax, -0x47(%rdi)
L(P7Q7): mov    %rax, -0x3f(%rdi)
L(P7Q6): mov    %rax, -0x37(%rdi)
L(P7Q5): mov    %rax, -0x2f(%rdi)
L(P7Q4): mov    %rax, -0x27(%rdi)
L(P7Q3): mov    %rax, -0x1f(%rdi)
L(P7Q2): mov    %rax, -0x17(%rdi)
L(P7Q1): mov    %rax, -0xf(%rdi)
L(P7Q0): mov    %eax, -0x7(%rdi)
         mov    %ax, -0x3(%rdi)
         mov    %al, -0x1(%rdi)
         ret

        /*
         * Align to a 16-byte boundary. Avoids penalties from unaligned stores
         * as well as from stores spanning cachelines. Note 16-byte alignment
         * is better in case where rep sstosq is used.
         */
        .p2align 4
L(ck_align):
        test    $0xf, %rdi
        jz      L(aligned_now)
        test    $1, %rdi
        jz      2f
        mov     %al, (%rdi)
        dec     %rsi
        lea     1(%rdi),%rdi
2:
        test    $2, %rdi
        jz      4f
        mov     %ax, (%rdi)
        sub     $2, %rsi
        lea     2(%rdi),%rdi
4:
        test    $4, %rdi
        jz      8f
        mov     %eax, (%rdi)
        sub     $4, %rsi
        lea     4(%rdi),%rdi
8:
        test    $8, %rdi
        jz      L(aligned_now)
        mov     %rax, (%rdi)
        sub     $8, %rsi
        lea     8(%rdi),%rdi

        /*
         * For large sizes rep sstoq is fastest.
         * Transition point determined experimentally as measured on
         * Intel Xeon processors (incl. Nehalem) and AMD Opteron.
         */
L(aligned_now):
        cmp     $BZERO_USE_REP, %rsi
        ja      L(use_rep)

        /*
         * zero 64-bytes per loop
         */
        .p2align 4
L(bzero_loop):
        leaq    -0x40(%rsi), %rsi
        cmpq    $0x40, %rsi
        movq    %rax, (%rdi)
        movq    %rax, 0x8(%rdi)
        movq    %rax, 0x10(%rdi)
        movq    %rax, 0x18(%rdi)
        movq    %rax, 0x20(%rdi)
        movq    %rax, 0x28(%rdi)
        movq    %rax, 0x30(%rdi)
        movq    %rax, 0x38(%rdi)
        leaq    0x40(%rdi), %rdi
        jae     L(bzero_loop)

        /*
         * Clear any remaining bytes..
         */
9:
        leaq    L(setPxQx)(%rip), %r10
        addq    %rsi, %rdi
        movslq  (%r10,%rsi,4), %rcx
        leaq    (%rcx,%r10,1), %r10
        INDIRECT_JMP_REG(r10)

        /*
         * Use rep sstoq. Clear any remainder via unrolled code
         */
        .p2align 4
L(use_rep):
        movq    %rsi, %rcx              /* get size in bytes */
        shrq    $3, %rcx                /* count of 8-byte words to zero */
        rep
          sstoq                         /* %rcx = words to clear (%rax=0) */
        andq    $7, %rsi                /* remaining bytes */
        jnz     9b
        ret
#undef  L
        SET_SIZE(bzero_altentry)
        SET_SIZE(bzero)

/*
 * Transfer data to and from user space -
 * Note that these routines can cause faults
 * It is assumed that the kernel has nothing at
 * less than KERNELBASE in the virtual address space.
 *
 * Note that copyin(9F) and copyout(9F) are part of the
 * DDI/DKI which specifies that they return '-1' on "errors."
 *
 * Sigh.
 *
 * So there's two extremely similar routines - xcopyin_nta() and
 * xcopyout_nta() which return the errno that we've faithfully computed.
 * This allows other callers (e.g. uiomove(9F)) to work correctly.
 * Given that these are used pretty heavily, we expand the calling
 * sequences inline for all flavours (rather than making wrappers).
 */

/*
 * Copy user data to kernel space.
 */

        ENTRY(copyin)
        pushq   %rbp
        movq    %rsp, %rbp
        subq    $24, %rsp

        /*
         * save args in case we trap and need to rerun as a copyop
         */
        movq    %rdi, (%rsp)
        movq    %rsi, 0x8(%rsp)
        movq    %rdx, 0x10(%rsp)

        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rsi              /* %rsi = kaddr */
        jnb     1f
        leaq    .copyin_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif
        /*
         * pass lofault value as 4th argument to do_copy_fault
         */
        leaq    _copyin_err(%rip), %rcx

        movq    %gs:CPU_THREAD, %r9
        cmpq    %rax, %rdi              /* test uaddr < kernelbase */
        jae     3f                      /* take copyop if uaddr > kernelbase */
        SMAP_DISABLE_INSTR(0)
        jmp     do_copy_fault           /* Takes care of leave for us */

_copyin_err:
        SMAP_ENABLE_INSTR(2)
        movq    %r11, T_LOFAULT(%r9)    /* restore original lofault */
        addq    $8, %rsp                /* pop bcopy_altentry call ret addr */
3:
        movq    T_COPYOPS(%r9), %rax
        cmpq    $0, %rax
        jz      2f
        /*
         * reload args for the copyop
         */
        movq    (%rsp), %rdi
        movq    0x8(%rsp), %rsi
        movq    0x10(%rsp), %rdx
        leave
        movq    CP_COPYIN(%rax), %rax
        INDIRECT_JMP_REG(rax)

2:      movl    $-1, %eax
        leave
        ret
        SET_SIZE(copyin)

        ENTRY(xcopyin_nta)
        pushq   %rbp
        movq    %rsp, %rbp
        subq    $24, %rsp

        /*
         * save args in case we trap and need to rerun as a copyop
         * %rcx is consumed in this routine so we don't need to save
         * it.
         */
        movq    %rdi, (%rsp)
        movq    %rsi, 0x8(%rsp)
        movq    %rdx, 0x10(%rsp)

        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rsi              /* %rsi = kaddr */
        jnb     1f
        leaq    .xcopyin_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif
        movq    %gs:CPU_THREAD, %r9
        cmpq    %rax, %rdi              /* test uaddr < kernelbase */
        jae     4f
        cmpq    $0, %rcx                /* No non-temporal access? */
        /*
         * pass lofault value as 4th argument to do_copy_fault
         */
        leaq    _xcopyin_err(%rip), %rcx        /* doesn't set rflags */
        jnz     6f                      /* use regular access */
        /*
         * Make sure cnt is >= XCOPY_MIN_SIZE bytes
         */
        cmpq    $XCOPY_MIN_SIZE, %rdx
        jae     5f
6:
        SMAP_DISABLE_INSTR(1)
        jmp     do_copy_fault

        /*
         * Make sure src and dst are NTA_ALIGN_SIZE aligned,
         * count is COUNT_ALIGN_SIZE aligned.
         */
5:
        movq    %rdi, %r10
        orq     %rsi, %r10
        andq    $NTA_ALIGN_MASK, %r10
        orq     %rdx, %r10
        andq    $COUNT_ALIGN_MASK, %r10
        jnz     6b
        leaq    _xcopyin_nta_err(%rip), %rcx    /* doesn't set rflags */
        SMAP_DISABLE_INSTR(2)
        jmp     do_copy_fault_nta       /* use non-temporal access */

4:
        movl    $EFAULT, %eax
        jmp     3f

        /*
         * A fault during do_copy_fault or do_copy_fault_nta is
         * indicated through an errno value in %rax and we iret from the
         * trap handler to here.
         */
_xcopyin_err:
        addq    $8, %rsp                /* pop bcopy_altentry call ret addr */
_xcopyin_nta_err:
        SMAP_ENABLE_INSTR(3)
        movq    %r11, T_LOFAULT(%r9)    /* restore original lofault */
3:
        movq    T_COPYOPS(%r9), %r8
        cmpq    $0, %r8
        jz      2f

        /*
         * reload args for the copyop
         */
        movq    (%rsp), %rdi
        movq    0x8(%rsp), %rsi
        movq    0x10(%rsp), %rdx
        leave
        movq    CP_XCOPYIN(%r8), %r8
        INDIRECT_JMP_REG(r8)

2:      leave
        ret
        SET_SIZE(xcopyin_nta)

/*
 * Copy kernel data to user space.
 */

        ENTRY(copyout)
        pushq   %rbp
        movq    %rsp, %rbp
        subq    $24, %rsp

        /*
         * save args in case we trap and need to rerun as a copyop
         */
        movq    %rdi, (%rsp)
        movq    %rsi, 0x8(%rsp)
        movq    %rdx, 0x10(%rsp)

        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rdi              /* %rdi = kaddr */
        jnb     1f
        leaq    .copyout_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif
        /*
         * pass lofault value as 4th argument to do_copy_fault
         */
        leaq    _copyout_err(%rip), %rcx

        movq    %gs:CPU_THREAD, %r9
        cmpq    %rax, %rsi              /* test uaddr < kernelbase */
        jae     3f                      /* take copyop if uaddr > kernelbase */
        SMAP_DISABLE_INSTR(3)
        jmp     do_copy_fault           /* Calls leave for us */

_copyout_err:
        SMAP_ENABLE_INSTR(4)
        movq    %r11, T_LOFAULT(%r9)    /* restore original lofault */
        addq    $8, %rsp                /* pop bcopy_altentry call ret addr */
3:
        movq    T_COPYOPS(%r9), %rax
        cmpq    $0, %rax
        jz      2f

        /*
         * reload args for the copyop
         */
        movq    (%rsp), %rdi
        movq    0x8(%rsp), %rsi
        movq    0x10(%rsp), %rdx
        leave
        movq    CP_COPYOUT(%rax), %rax
        INDIRECT_JMP_REG(rax)

2:      movl    $-1, %eax
        leave
        ret
        SET_SIZE(copyout)

        ENTRY(xcopyout_nta)
        pushq   %rbp
        movq    %rsp, %rbp
        subq    $24, %rsp

        /*
         * save args in case we trap and need to rerun as a copyop
         */
        movq    %rdi, (%rsp)
        movq    %rsi, 0x8(%rsp)
        movq    %rdx, 0x10(%rsp)

        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rdi              /* %rdi = kaddr */
        jnb     1f
        leaq    .xcopyout_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif
        movq    %gs:CPU_THREAD, %r9
        cmpq    %rax, %rsi              /* test uaddr < kernelbase */
        jae     4f

        cmpq    $0, %rcx                /* No non-temporal access? */
        /*
         * pass lofault value as 4th argument to do_copy_fault
         */
        leaq    _xcopyout_err(%rip), %rcx
        jnz     6f
        /*
         * Make sure cnt is >= XCOPY_MIN_SIZE bytes
         */
        cmpq    $XCOPY_MIN_SIZE, %rdx
        jae     5f
6:
        SMAP_DISABLE_INSTR(4)
        jmp     do_copy_fault

        /*
         * Make sure src and dst are NTA_ALIGN_SIZE aligned,
         * count is COUNT_ALIGN_SIZE aligned.
         */
5:
        movq    %rdi, %r10
        orq     %rsi, %r10
        andq    $NTA_ALIGN_MASK, %r10
        orq     %rdx, %r10
        andq    $COUNT_ALIGN_MASK, %r10
        jnz     6b
        leaq    _xcopyout_nta_err(%rip), %rcx
        SMAP_DISABLE_INSTR(5)
        call    do_copy_fault_nta
        SMAP_ENABLE_INSTR(5)
        ret

4:
        movl    $EFAULT, %eax
        jmp     3f

        /*
         * A fault during do_copy_fault or do_copy_fault_nta is
         * indicated through an errno value in %rax and we iret from the
         * trap handler to here.
         */
_xcopyout_err:
        addq    $8, %rsp                /* pop bcopy_altentry call ret addr */
_xcopyout_nta_err:
        SMAP_ENABLE_INSTR(6)
        movq    %r11, T_LOFAULT(%r9)    /* restore original lofault */
3:
        movq    T_COPYOPS(%r9), %r8
        cmpq    $0, %r8
        jz      2f

        /*
         * reload args for the copyop
         */
        movq    (%rsp), %rdi
        movq    0x8(%rsp), %rsi
        movq    0x10(%rsp), %rdx
        leave
        movq    CP_XCOPYOUT(%r8), %r8
        INDIRECT_JMP_REG(r8)

2:      leave
        ret
        SET_SIZE(xcopyout_nta)

/*
 * Copy a null terminated string from one point to another in
 * the kernel address space.
 */

        ENTRY(copystr)
        pushq   %rbp
        movq    %rsp, %rbp
#ifdef DEBUG
        movq    kernelbase(%rip), %rax
        cmpq    %rax, %rdi              /* %rdi = from */
        jb      0f
        cmpq    %rax, %rsi              /* %rsi = to */
        jnb     1f
0:      leaq    .copystr_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif
        movq    %gs:CPU_THREAD, %r9
        movq    T_LOFAULT(%r9), %r8     /* pass current lofault value as */
                                        /* 5th argument to do_copystr */
        xorl    %r10d,%r10d             /* pass smap restore need in %r10d */
                                        /* as a non-ABI 6th arg */
do_copystr:
        movq    %gs:CPU_THREAD, %r9     /* %r9 = thread addr */
        movq    T_LOFAULT(%r9), %r11    /* save the current lofault */
        movq    %r8, T_LOFAULT(%r9)     /* new lofault */

        movq    %rdx, %r8               /* save maxlength */

        cmpq    $0, %rdx                /* %rdx = maxlength */
        je      copystr_enametoolong    /* maxlength == 0 */

copystr_loop:
        decq    %r8
        movb    (%rdi), %al
        incq    %rdi
        movb    %al, (%rsi)
        incq    %rsi
        cmpb    $0, %al
        je      copystr_null            /* null char */
        cmpq    $0, %r8
        jne     copystr_loop

copystr_enametoolong:
        movl    $ENAMETOOLONG, %eax
        jmp     copystr_out

copystr_null:
        xorl    %eax, %eax              /* no error */

copystr_out:
        cmpq    $0, %rcx                /* want length? */
        je      copystr_smap            /* no */
        subq    %r8, %rdx               /* compute length and store it */
        movq    %rdx, (%rcx)

copystr_smap:
        cmpl    $0, %r10d
        jz      copystr_done
        SMAP_ENABLE_INSTR(7)

copystr_done:
        movq    %r11, T_LOFAULT(%r9)    /* restore the original lofault */
        leave
        ret
        SET_SIZE(copystr)

/*
 * Copy a null terminated string from the user address space into
 * the kernel address space.
 */

        ENTRY(copyinstr)
        pushq   %rbp
        movq    %rsp, %rbp
        subq    $32, %rsp

        /*
         * save args in case we trap and need to rerun as a copyop
         */
        movq    %rdi, (%rsp)
        movq    %rsi, 0x8(%rsp)
        movq    %rdx, 0x10(%rsp)
        movq    %rcx, 0x18(%rsp)

        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rsi              /* %rsi = kaddr */
        jnb     1f
        leaq    .copyinstr_panic_msg(%rip), %rdi
        xorl    %eax, %eax
        call    panic
1:
#endif
        /*
         * pass lofault value as 5th argument to do_copystr
         * do_copystr expects whether or not we need smap in %r10d
         */
        leaq    _copyinstr_error(%rip), %r8
        movl    $1, %r10d

        cmpq    %rax, %rdi              /* test uaddr < kernelbase */
        jae     4f
        SMAP_DISABLE_INSTR(6)
        jmp     do_copystr
4:
        movq    %gs:CPU_THREAD, %r9
        jmp     3f

_copyinstr_error:
        SMAP_ENABLE_INSTR(8)
        movq    %r11, T_LOFAULT(%r9)    /* restore original lofault */
3:
        movq    T_COPYOPS(%r9), %rax
        cmpq    $0, %rax
        jz      2f

        /*
         * reload args for the copyop
         */
        movq    (%rsp), %rdi
        movq    0x8(%rsp), %rsi
        movq    0x10(%rsp), %rdx
        movq    0x18(%rsp), %rcx
        leave
        movq    CP_COPYINSTR(%rax), %rax
        INDIRECT_JMP_REG(rax)

2:      movl    $EFAULT, %eax           /* return EFAULT */
        leave
        ret
        SET_SIZE(copyinstr)

/*
 * Copy a null terminated string from the kernel
 * address space to the user address space.
 */

        ENTRY(copyoutstr)
        pushq   %rbp
        movq    %rsp, %rbp
        subq    $32, %rsp

        /*
         * save args in case we trap and need to rerun as a copyop
         */
        movq    %rdi, (%rsp)
        movq    %rsi, 0x8(%rsp)
        movq    %rdx, 0x10(%rsp)
        movq    %rcx, 0x18(%rsp)

        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rdi              /* %rdi = kaddr */
        jnb     1f
        leaq    .copyoutstr_panic_msg(%rip), %rdi
        jmp     call_panic              /* setup stack and call panic */
1:
#endif
        /*
         * pass lofault value as 5th argument to do_copystr
         * pass one as 6th argument to do_copystr in %r10d
         */
        leaq    _copyoutstr_error(%rip), %r8
        movl    $1, %r10d

        cmpq    %rax, %rsi              /* test uaddr < kernelbase */
        jae     4f
        SMAP_DISABLE_INSTR(7)
        jmp     do_copystr
4:
        movq    %gs:CPU_THREAD, %r9
        jmp     3f

_copyoutstr_error:
        SMAP_ENABLE_INSTR(9)
        movq    %r11, T_LOFAULT(%r9)    /* restore the original lofault */
3:
        movq    T_COPYOPS(%r9), %rax
        cmpq    $0, %rax
        jz      2f

        /*
         * reload args for the copyop
         */
        movq    (%rsp), %rdi
        movq    0x8(%rsp), %rsi
        movq    0x10(%rsp), %rdx
        movq    0x18(%rsp), %rcx
        leave
        movq    CP_COPYOUTSTR(%rax), %rax
        INDIRECT_JMP_REG(rax)

2:      movl    $EFAULT, %eax           /* return EFAULT */
        leave
        ret
        SET_SIZE(copyoutstr)

/*
 * Since all of the fuword() variants are so similar, we have a macro to spit
 * them out.  This allows us to create DTrace-unobservable functions easily.
 */

/*
 * Note that we don't save and reload the arguments here
 * because their values are not altered in the copy path.
 * Additionally, when successful, the smap_enable jmp will
 * actually return us to our original caller.
 */

#define FUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2)      \
        ENTRY(NAME)                             \
        movq    %gs:CPU_THREAD, %r9;            \
        cmpq    kernelbase(%rip), %rdi;         \
        jae     1f;                             \
        leaq    _flt_##NAME, %rdx;              \
        movq    %rdx, T_LOFAULT(%r9);           \
        SMAP_DISABLE_INSTR(DISNUM)              \
        INSTR   (%rdi), REG;                    \
        movq    $0, T_LOFAULT(%r9);             \
        INSTR   REG, (%rsi);                    \
        xorl    %eax, %eax;                     \
        SMAP_ENABLE_INSTR(EN1)                  \
        ret;                                    \
_flt_##NAME:                                    \
        SMAP_ENABLE_INSTR(EN2)                  \
        movq    $0, T_LOFAULT(%r9);             \
1:                                              \
        movq    T_COPYOPS(%r9), %rax;           \
        cmpq    $0, %rax;                       \
        jz      2f;                             \
        movq    COPYOP(%rax), %rax;             \
        INDIRECT_JMP_REG(rax);                  \
2:                                              \
        movl    $-1, %eax;                      \
        ret;                                    \
        SET_SIZE(NAME)

        FUWORD(fuword64, movq, %rax, CP_FUWORD64,8,10,11)
        FUWORD(fuword32, movl, %eax, CP_FUWORD32,9,12,13)
        FUWORD(fuword16, movw, %ax, CP_FUWORD16,10,14,15)
        FUWORD(fuword8, movb, %al, CP_FUWORD8,11,16,17)

#undef  FUWORD

/*
 * Set user word.
 */

/*
 * Note that we don't save and reload the arguments here
 * because their values are not altered in the copy path.
 */

#define SUWORD(NAME, INSTR, REG, COPYOP, DISNUM, EN1, EN2)      \
        ENTRY(NAME)                             \
        movq    %gs:CPU_THREAD, %r9;            \
        cmpq    kernelbase(%rip), %rdi;         \
        jae     1f;                             \
        leaq    _flt_##NAME, %rdx;              \
        SMAP_DISABLE_INSTR(DISNUM)              \
        movq    %rdx, T_LOFAULT(%r9);           \
        INSTR   REG, (%rdi);                    \
        movq    $0, T_LOFAULT(%r9);             \
        xorl    %eax, %eax;                     \
        SMAP_ENABLE_INSTR(EN1)                  \
        ret;                                    \
_flt_##NAME:                                    \
        SMAP_ENABLE_INSTR(EN2)                  \
        movq    $0, T_LOFAULT(%r9);             \
1:                                              \
        movq    T_COPYOPS(%r9), %rax;           \
        cmpq    $0, %rax;                       \
        jz      3f;                             \
        movq    COPYOP(%rax), %rax;             \
        INDIRECT_JMP_REG(rax);                  \
3:                                              \
        movl    $-1, %eax;                      \
        ret;                                    \
        SET_SIZE(NAME)

        SUWORD(suword64, movq, %rsi, CP_SUWORD64,12,18,19)
        SUWORD(suword32, movl, %esi, CP_SUWORD32,13,20,21)
        SUWORD(suword16, movw, %si, CP_SUWORD16,14,22,23)
        SUWORD(suword8, movb, %sil, CP_SUWORD8,15,24,25)

#undef  SUWORD

#define FUWORD_NOERR(NAME, INSTR, REG)          \
        ENTRY(NAME)                             \
        cmpq    kernelbase(%rip), %rdi;         \
        cmovnbq kernelbase(%rip), %rdi;         \
        INSTR   (%rdi), REG;                    \
        INSTR   REG, (%rsi);                    \
        ret;                                    \
        SET_SIZE(NAME)

        FUWORD_NOERR(fuword64_noerr, movq, %rax)
        FUWORD_NOERR(fuword32_noerr, movl, %eax)
        FUWORD_NOERR(fuword16_noerr, movw, %ax)
        FUWORD_NOERR(fuword8_noerr, movb, %al)

#undef  FUWORD_NOERR

#define SUWORD_NOERR(NAME, INSTR, REG)          \
        ENTRY(NAME)                             \
        cmpq    kernelbase(%rip), %rdi;         \
        cmovnbq kernelbase(%rip), %rdi;         \
        INSTR   REG, (%rdi);                    \
        ret;                                    \
        SET_SIZE(NAME)

        SUWORD_NOERR(suword64_noerr, movq, %rsi)
        SUWORD_NOERR(suword32_noerr, movl, %esi)
        SUWORD_NOERR(suword16_noerr, movw, %si)
        SUWORD_NOERR(suword8_noerr, movb, %sil)

#undef  SUWORD_NOERR


        .weak   subyte
        subyte=suword8
        .weak   subyte_noerr
        subyte_noerr=suword8_noerr

        .weak   fulword
        fulword=fuword64
        .weak   fulword_noerr
        fulword_noerr=fuword64_noerr
        .weak   sulword
        sulword=suword64
        .weak   sulword_noerr
        sulword_noerr=suword64_noerr

        ENTRY(copyin_noerr)
        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rsi              /* %rsi = kto */
        jae     1f
        leaq    .cpyin_ne_pmsg(%rip), %rdi
        jmp     call_panic              /* setup stack and call panic */
1:
#endif
        cmpq    %rax, %rdi              /* ufrom < kernelbase */
        jb      do_copy
        movq    %rax, %rdi              /* force fault at kernelbase */
        jmp     do_copy
        SET_SIZE(copyin_noerr)

        ENTRY(copyout_noerr)
        movq    kernelbase(%rip), %rax
#ifdef DEBUG
        cmpq    %rax, %rdi              /* %rdi = kfrom */
        jae     1f
        leaq    .cpyout_ne_pmsg(%rip), %rdi
        jmp     call_panic              /* setup stack and call panic */
1:
#endif
        cmpq    %rax, %rsi              /* uto < kernelbase */
        jb      do_copy
        movq    %rax, %rsi              /* force fault at kernelbase */
        jmp     do_copy
        SET_SIZE(copyout_noerr)

        ENTRY(uzero)
        movq    kernelbase(%rip), %rax
        cmpq    %rax, %rdi
        jb      do_zero
        movq    %rax, %rdi      /* force fault at kernelbase */
        jmp     do_zero
        SET_SIZE(uzero)

        ENTRY(ucopy)
        movq    kernelbase(%rip), %rax
        cmpq    %rax, %rdi
        cmovaeq %rax, %rdi      /* force fault at kernelbase */
        cmpq    %rax, %rsi
        cmovaeq %rax, %rsi      /* force fault at kernelbase */
        jmp     do_copy
        SET_SIZE(ucopy)

        /*
         * Note, the frame pointer is required here becuase do_copystr expects
         * to be able to pop it off!
         */
        ENTRY(ucopystr)
        pushq   %rbp
        movq    %rsp, %rbp
        movq    kernelbase(%rip), %rax
        cmpq    %rax, %rdi
        cmovaeq %rax, %rdi      /* force fault at kernelbase */
        cmpq    %rax, %rsi
        cmovaeq %rax, %rsi      /* force fault at kernelbase */
        /* do_copystr expects lofault address in %r8 */
        /* do_copystr expects whether or not we need smap in %r10 */
        xorl    %r10d, %r10d
        movq    %gs:CPU_THREAD, %r8
        movq    T_LOFAULT(%r8), %r8
        jmp     do_copystr
        SET_SIZE(ucopystr)

#ifdef DEBUG
        .data
.kcopy_panic_msg:
        .string "kcopy: arguments below kernelbase"
.bcopy_panic_msg:
        .string "bcopy: arguments below kernelbase"
.kzero_panic_msg:
        .string "kzero: arguments below kernelbase"
.bzero_panic_msg:
        .string "bzero: arguments below kernelbase"
.copyin_panic_msg:
        .string "copyin: kaddr argument below kernelbase"
.xcopyin_panic_msg:
        .string "xcopyin: kaddr argument below kernelbase"
.copyout_panic_msg:
        .string "copyout: kaddr argument below kernelbase"
.xcopyout_panic_msg:
        .string "xcopyout: kaddr argument below kernelbase"
.copystr_panic_msg:
        .string "copystr: arguments in user space"
.copyinstr_panic_msg:
        .string "copyinstr: kaddr argument not in kernel address space"
.copyoutstr_panic_msg:
        .string "copyoutstr: kaddr argument not in kernel address space"
.cpyin_ne_pmsg:
        .string "copyin_noerr: argument not in kernel address space"
.cpyout_ne_pmsg:
        .string "copyout_noerr: argument not in kernel address space"
#endif

.data
.align  4
.globl  _smap_enable_patch_count
.type   _smap_enable_patch_count,@object
.size   _smap_enable_patch_count, 4
_smap_enable_patch_count:
        .long   SMAP_ENABLE_COUNT

.globl  _smap_disable_patch_count
.type   _smap_disable_patch_count,@object
.size   _smap_disable_patch_count, 4
_smap_disable_patch_count:
        .long SMAP_DISABLE_COUNT