/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AMD Memory Encryption Support
 *
 * Copyright (C) 2016 Advanced Micro Devices, Inc.
 *
 * Author: Tom Lendacky <thomas.lendacky@amd.com>
 */

#include <linux/linkage.h>
#include <linux/pgtable.h>
#include <asm/page.h>
#include <asm/processor-flags.h>
#include <asm/msr-index.h>
#include <asm/nospec-branch.h>

        .text
        .code64
SYM_FUNC_START(__pi_sme_encrypt_execute)

        /*
         * Entry parameters:
         *   RDI - virtual address for the encrypted mapping
         *   RSI - virtual address for the decrypted mapping
         *   RDX - length to encrypt
         *   RCX - virtual address of the encryption workarea, including:
         *     - stack page (PAGE_SIZE)
         *     - encryption routine page (PAGE_SIZE)
         *     - intermediate copy buffer (PMD_SIZE)
         *    R8 - physical address of the pagetables to use for encryption
         */

        push    %rbp
        movq    %rsp, %rbp              /* RBP now has original stack pointer */

        /* Set up a one page stack in the non-encrypted memory area */
        movq    %rcx, %rax              /* Workarea stack page */
        leaq    PAGE_SIZE(%rax), %rsp   /* Set new stack pointer */
        addq    $PAGE_SIZE, %rax        /* Workarea encryption routine */

        push    %r12
        movq    %rdi, %r10              /* Encrypted area */
        movq    %rsi, %r11              /* Decrypted area */
        movq    %rdx, %r12              /* Area length */

        /* Copy encryption routine into the workarea */
        movq    %rax, %rdi                              /* Workarea encryption routine */
        leaq    __enc_copy(%rip), %rsi                  /* Encryption routine */
        movq    $(.L__enc_copy_end - __enc_copy), %rcx  /* Encryption routine length */
        rep     movsb

        /* Setup registers for call */
        movq    %r10, %rdi              /* Encrypted area */
        movq    %r11, %rsi              /* Decrypted area */
        movq    %r8, %rdx               /* Pagetables used for encryption */
        movq    %r12, %rcx              /* Area length */
        movq    %rax, %r8               /* Workarea encryption routine */
        addq    $PAGE_SIZE, %r8         /* Workarea intermediate copy buffer */

        ANNOTATE_RETPOLINE_SAFE
        call    *%rax                   /* Call the encryption routine */

        pop     %r12

        movq    %rbp, %rsp              /* Restore original stack pointer */
        pop     %rbp

        /* Offset to __x86_return_thunk would be wrong here */
        ANNOTATE_UNRET_SAFE
        ret
        int3
SYM_FUNC_END(__pi_sme_encrypt_execute)

SYM_FUNC_START_LOCAL(__enc_copy)
        ANNOTATE_NOENDBR
/*
 * Routine used to encrypt memory in place.
 *   This routine must be run outside of the kernel proper since
 *   the kernel will be encrypted during the process. So this
 *   routine is defined here and then copied to an area outside
 *   of the kernel where it will remain and run decrypted
 *   during execution.
 *
 *   On entry the registers must be:
 *     RDI - virtual address for the encrypted mapping
 *     RSI - virtual address for the decrypted mapping
 *     RDX - address of the pagetables to use for encryption
 *     RCX - length of area
 *      R8 - intermediate copy buffer
 *
 *     RAX - points to this routine
 *
 * The area will be encrypted by copying from the non-encrypted
 * memory space to an intermediate buffer and then copying from the
 * intermediate buffer back to the encrypted memory space. The physical
 * addresses of the two mappings are the same which results in the area
 * being encrypted "in place".
 */
        /* Enable the new page tables */
        mov     %rdx, %cr3

        /* Flush any global TLBs */
        mov     %cr4, %rdx
        andq    $~X86_CR4_PGE, %rdx
        mov     %rdx, %cr4
        orq     $X86_CR4_PGE, %rdx
        mov     %rdx, %cr4

        push    %r15
        push    %r12

        movq    %rcx, %r9               /* Save area length */
        movq    %rdi, %r10              /* Save encrypted area address */
        movq    %rsi, %r11              /* Save decrypted area address */

        /* Set the PAT register PA5 entry to write-protect */
        movl    $MSR_IA32_CR_PAT, %ecx
        rdmsr
        mov     %rdx, %r15              /* Save original PAT value */
        andl    $0xffff00ff, %edx       /* Clear PA5 */
        orl     $0x00000500, %edx       /* Set PA5 to WP */
        wrmsr

        wbinvd                          /* Invalidate any cache entries */

        /* Copy/encrypt up to 2MB at a time */
        movq    $PMD_SIZE, %r12
1:
        cmpq    %r12, %r9
        jnb     2f
        movq    %r9, %r12

2:
        movq    %r11, %rsi              /* Source - decrypted area */
        movq    %r8, %rdi               /* Dest   - intermediate copy buffer */
        movq    %r12, %rcx
        rep     movsb

        movq    %r8, %rsi               /* Source - intermediate copy buffer */
        movq    %r10, %rdi              /* Dest   - encrypted area */
        movq    %r12, %rcx
        rep     movsb

        addq    %r12, %r11
        addq    %r12, %r10
        subq    %r12, %r9               /* Kernel length decrement */
        jnz     1b                      /* Kernel length not zero? */

        /* Restore PAT register */
        movl    $MSR_IA32_CR_PAT, %ecx
        rdmsr
        mov     %r15, %rdx              /* Restore original PAT value */
        wrmsr

        pop     %r12
        pop     %r15

        /* Offset to __x86_return_thunk would be wrong here */
        ANNOTATE_UNRET_SAFE
        ret
        int3
.L__enc_copy_end:
SYM_FUNC_END(__enc_copy)