/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 */

/*
 * Assembly variants of various functions, for those that don't need generic C
 * implementations.  Currently this includes:
 *
 * - Direct-access versions of copyin/copyout methods.
 *   - These are used by Radix AIM pmap (ISA 3.0), and Book-E, to avoid
 *     unnecessary pmap_map_usr_ptr() calls.
 */

#include "assym.inc"
#include "opt_sched.h"

#include <sys/syscall.h>
#include <sys/errno.h>
        
#include <machine/param.h>
#include <machine/asm.h>
#include <machine/spr.h>
#include <machine/trap.h>
#include <machine/vmparam.h>

#ifdef _CALL_ELF
.abiversion _CALL_ELF
#endif

#ifdef __powerpc64__
#define LOAD    ld
#define STORE   std
#define WORD    8
#define CMPI    cmpdi
#define CMPLI   cmpldi
/* log_2(8 * WORD) */
#define LOOP_LOG        6
#define LOG_WORD        3
#define CURTHREAD       %r13
#else
#define LOAD    lwz
#define STORE   stw
#define WORD    4
#define CMPI    cmpwi
#define CMPLI   cmplwi
/* log_2(8 * WORD) */
#define LOOP_LOG        5
#define LOG_WORD        2
#define CURTHREAD       %r2
#endif

#ifdef AIM
#define ENTRY_DIRECT(x) ENTRY(x ## _direct)
#define END_DIRECT(x)   END(x ## _direct)
#else
#define ENTRY_DIRECT(x) ENTRY(x)
#define END_DIRECT(x)   END(x)
#endif
        
#ifdef __powerpc64__
#define PROLOGUE                ;\
        mflr    %r0             ;\
        std     %r0, 16(%r1)    ;\

#define EPILOGUE                ;\
        ld      %r0, 16(%r1)    ;\
        mtlr    %r0             ;\
        blr                     ;\
        nop

#define VALIDATE_TRUNCATE_ADDR_COPY     VALIDATE_ADDR_COPY
#define VALIDATE_ADDR_COPY(raddr, len)  \
        srdi  %r0, raddr, 52            ;\
        cmpwi %r0, 1                    ;\
        bge-    copy_fault              ;\
        nop

#define VALIDATE_ADDR_FUSU(raddr)       ;\
        srdi  %r0, raddr, 52            ;\
        cmpwi %r0, 1                    ;\
        bge-    fusufault               ;\
        nop

#else
#define PROLOGUE                ;\
        mflr    %r0             ;\
        stw     %r0, 4(%r1)     ;\

#define EPILOGUE                ;\
        lwz     %r0, 4(%r1)     ;\
        mtlr    %r0             ;\
        blr                     ;\
        nop

/* %r0 is temporary */
/*
 * Validate address and length are valid.
 * For VALIDATE_ADDR_COPY() have to account for wraparound.
 */
#define VALIDATE_ADDR_COPY(raddr, len)          \
        lis     %r0, VM_MAXUSER_ADDRESS@h       ;\
        ori     %r0, %r0, VM_MAXUSER_ADDRESS@l  ;\
        cmplw   %r0, raddr                      ;\
        blt-    copy_fault                      ;\
        add     %r0, raddr, len                 ;\
        cmplw   7, %r0, raddr                   ;\
        blt-    7, copy_fault                   ;\
        mtcrf   0x80, %r0                       ;\
        bt-     0, copy_fault                   ;\
        nop

#define VALIDATE_TRUNCATE_ADDR_COPY(raddr, len)         \
        lis     %r0, VM_MAXUSER_ADDRESS@h       ;\
        ori     %r0, %r0, VM_MAXUSER_ADDRESS@l  ;\
        cmplw   %r0, raddr                      ;\
        blt-    copy_fault                      ;\
        sub     %r0, %r0, raddr                 ;\
        cmplw   len, %r0                        ;\
        isel    len, len, %r0, 0                ;\

#define VALIDATE_ADDR_FUSU(raddr)               \
        lis     %r0, VM_MAXUSER_ADDRESS@h       ;\
        ori     %r0, %r0, VM_MAXUSER_ADDRESS@l  ;\
        cmplw   %r0, raddr                      ;\
        ble-    fusufault

#endif

#define SET_COPYFAULT(raddr, rpcb, len) \
        VALIDATE_ADDR_COPY(raddr, len)  ;\
        li      %r0, COPYFAULT          ;\
        LOAD    rpcb, TD_PCB(CURTHREAD) ;\
        STORE   %r0, PCB_ONFAULT(rpcb)  ;\

#define SET_COPYFAULT_TRUNCATE(raddr, rpcb, len)\
        VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) ;\
        li      %r0, COPYFAULT                  ;\
        LOAD    rpcb, TD_PCB(CURTHREAD)         ;\
        STORE   %r0, PCB_ONFAULT(rpcb)

#define SET_FUSUFAULT(raddr, rpcb)      \
        VALIDATE_ADDR_FUSU(raddr)       ;\
        li      %r0, FUSUFAULT          ;\
        LOAD    rpcb, TD_PCB(CURTHREAD) ;\
        STORE   %r0, PCB_ONFAULT(rpcb)

#define CLEAR_FAULT_NO_CLOBBER(rpcb)    \
        LOAD    rpcb, TD_PCB(CURTHREAD) ;\
        li      %r0, 0                  ;\
        STORE   %r0, PCB_ONFAULT(rpcb)

#define CLEAR_FAULT(rpcb)               \
        CLEAR_FAULT_NO_CLOBBER(rpcb)    ;\
        li      %r3, 0

/*
 *  bcopy(src, dst, len)
 *        %r3  %r4  %r5
 * 
 *  %r7 is the pcb pointer
 * 
 *  %r0 and %r8-%r10 are volatile
 *  %r11 and %r12 are generally volatile, used in linking and exception
 *  handling.  Can be clobbered here.
 *
 * Does not allocate or use stack space, but clobbers all volatile registers.
 */

#define rs      %r3
#define rd      %r4
#define rl      %r5

#define t1      %r6
#define t2      %r7
#define t3      %r8
#define t4      %r9
#define t5      %r10
#define t6      %r11
#define t7      %r12
#define t8      %r0

#define Thresh  WORD * 8
#define W4      3
#define W2      2
#define W1      1
#define WORDS(n)        (32 - LOG_WORD - W##n)
.text
ENTRY(bcopy_generic)
        CMPLI   0, %r5, 0
        beq     .Lend
        dcbtst  0, rd
        dcbt    0, rs
        CMPLI   rl, Thresh
        blt     .Lsmall
        b       .Llarge
/* memcpy */
/* ... */
.Lsmall:                                /* < 8 words remaining */
        mtcrf   0x3, rl
.Lsmall_start:
        bf      WORDS(4), 0f
        LOAD    t1, 0(rs)
        LOAD    t2, WORD*1(rs)
        LOAD    t3, WORD*2(rs)
        LOAD    t4, WORD*3(rs)
        addi    rs, rs, WORD*4
        STORE   t1, 0(rd)
        STORE   t2, WORD*1(rd)
        STORE   t3, WORD*2(rd)
        STORE   t4, WORD*3(rd)
        addi    rd, rd, WORD*4
0:                                      /* < 4 words remaining */
        bf      WORDS(2), 1f
        LOAD    t1, 0(rs)
        LOAD    t2, WORD*1(rs)
        addi    rs, rs, WORD*2
        STORE   t1, 0(rd)
        STORE   t2, WORD*1(rd)
        addi    rd, rd, WORD*2
1:                                      /* < 2 words remaining */
        bf      WORDS(1), 2f
        LOAD    t1, 0(rs)
        addi    rs, rs, WORD
        STORE   t1, 0(rd)
        addi    rd, rd, WORD
2:                                      /* < 1 word remaining */
#ifdef __powerpc64__
        bf      29, 3f
        lwz     t1, 0(rs)
        addi    rs, rs, 4
        stw     t1, 0(rd)
        addi    rd, rd, 4
3:                                      /* < 4 bytes remaining */
#endif
        bf      30, 4f
        lhz     t1, 0(rs)
        addi    rs, rs, 2
        sth     t1, 0(rd)
        addi    rd, rd, 2
4:                                      /* < 2 bytes remaining */
        bf      31, .Lout
        lbz     t1, 0(rs)
        addi    rs, rs, 1
        stb     t1, 0(rd)
        addi    rd, rd, 1
        b       .Lout

        .align 4
.Llarge:
        neg     t3, rd
        andi.   t6, t3, WORD-1          /* Align rd to word size */
        mtctr   t6
        sub     rl, rl, t6
        beq+    .Llargealigned
1:
        lbz     t1, 0(rs)
        addi    rs, rs, 1
        stb     t1, 0(rd)
        addi    rd, rd, 1
        bdnz    1b

.Llargealigned:
        srwi.   t2, rl, LOOP_LOG  /* length >> log_2(loop_size) => 8W iterations */
        mtcrf   0x3, rl
        beq     .Lsmall_start
        mtctr   t2
        b       1f

        .align 5
1:
        LOAD    t1, 0(rs)
        LOAD    t2, WORD(rs)
        LOAD    t3, WORD*2(rs)
        LOAD    t4, WORD*3(rs)
        LOAD    t5, WORD*4(rs)
        LOAD    t6, WORD*5(rs)
        LOAD    t7, WORD*6(rs)
        LOAD    t8, WORD*7(rs)
        addi    rs, rs, WORD*8
        STORE   t1, 0(rd)
        STORE   t2, WORD*1(rd)
        STORE   t3, WORD*2(rd)
        STORE   t4, WORD*3(rd)
        STORE   t5, WORD*4(rd)
        STORE   t6, WORD*5(rd)
        STORE   t7, WORD*6(rd)
        STORE   t8, WORD*7(rd)
        addi    rd, rd, WORD*8
        bdnz    1b

        b       .Lsmall_start
.Lout:
/* done */      
.Lend:  
        blr
END(bcopy_generic)

/*
 * copyout(from_kernel, to_user, len)
 *         %r3,        %r4,    %r5
 */
ENTRY_DIRECT(copyout)
        PROLOGUE
        SET_COPYFAULT(%r4, %r7, %r5)
        bl bcopy_generic 
        nop
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(copyout)

/*
 * copyin(from_user, to_kernel, len)
 *        %r3,        %r4,    %r5
 */
ENTRY_DIRECT(copyin)
        PROLOGUE
        SET_COPYFAULT(%r3, %r7, %r5)
        bl bcopy_generic
        nop
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(copyin)

/*
 * copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
 *                      %r3          %r4         %r5        %r6 
 */
        
ENTRY_DIRECT(copyinstr)
        PROLOGUE
        SET_COPYFAULT_TRUNCATE(%r3, %r7, %r5)
        addi    %r9, %r5, 1
        mtctr   %r9
        mr      %r8, %r3
        addi    %r8, %r8, -1
        addi    %r4, %r4, -1
        li      %r3, ENAMETOOLONG
0:
        bdz-    2f
        lbzu    %r0, 1(%r8)
        stbu    %r0, 1(%r4)

        // NULL byte reached ?
        CMPI    %r0, 0
        beq-    1f
        b       0b
1:
        li      %r3, 0
2:
        /* skip storing length if done is NULL */
        CMPI    %r6, 0
        beq-    3f
        mfctr   %r0
        sub     %r0, %r9, %r0
        STORE   %r0, 0(%r6)
3:
        CLEAR_FAULT_NO_CLOBBER(%r7)
        EPILOGUE
END_DIRECT(copyinstr)

ENTRY_DIRECT(subyte)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        stb  %r4, 0(%r3)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(subyte)

#ifndef __powerpc64__
ENTRY_DIRECT(suword)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        stw  %r4, 0(%r3)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(suword)
#endif  

ENTRY_DIRECT(suword16)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        sth  %r4, 0(%r3)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(suword16)

ENTRY_DIRECT(suword32)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        stw  %r4, 0(%r3)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(suword32)

#ifdef __powerpc64__    
ENTRY_DIRECT(suword64)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        std  %r4, 0(%r3) 
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(suword64)

ENTRY_DIRECT(suword)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        std  %r4, 0(%r3) 
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(suword)
#endif  
        
ENTRY_DIRECT(fubyte)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        lbz %r3, 0(%r3)
        CLEAR_FAULT_NO_CLOBBER(%r7)
        EPILOGUE
END_DIRECT(fubyte)

ENTRY_DIRECT(fuword16)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        lhz %r3, 0(%r3)
        CLEAR_FAULT_NO_CLOBBER(%r7)
        EPILOGUE
END_DIRECT(fuword16)

#ifndef __powerpc64__
ENTRY_DIRECT(fueword)   
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        lwz  %r0, 0(%r3)
        stw  %r0,  0(%r4)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(fueword)     
#endif  
ENTRY_DIRECT(fueword32)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        lwz  %r0, 0(%r3)
        stw  %r0,  0(%r4)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(fueword32)

#ifdef __powerpc64__
ENTRY_DIRECT(fueword)   
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        ld  %r0, 0(%r3)
        std %r0, 0(%r4)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(fueword)     

ENTRY_DIRECT(fueword64)
        PROLOGUE
        SET_FUSUFAULT(%r3, %r7)
        ld  %r0, 0(%r3)
        std %r0, 0(%r4)
        CLEAR_FAULT(%r7)
        EPILOGUE
END_DIRECT(fueword64)
#endif

/*
 * casueword(volatile u_long *base, u_long old, u_long *oldp, u_long new)
 *                            %r3          %r4           %r5         %r6 
 */

#define CASUEWORD32(raddr, rpcb)                                        ;\
        PROLOGUE                                                        ;\
        SET_FUSUFAULT(raddr, rpcb)                                      ;\
        li      %r8, 0                                                  ;\
1:                                                                      ;\
        lwarx   %r0, 0, %r3                                             ;\
        cmplw   %r4, %r0                                                ;\
        bne     2f                                                      ;\
        stwcx.  %r6, 0, %r3                                             ;\
        bne-    3f                                                      ;\
        b       4f                                                      ;\
2:                                                                      ;\
        stwcx.  %r0, 0, %r3             /* clear reservation (74xx) */  ;\
3:                                                                      ;\
        li      %r8, 1                                                  ;\
4:                                                                      ;\
        stw     %r0, 0(%r5)                                             ;\
        CLEAR_FAULT_NO_CLOBBER(rpcb)                                    ;\
        mr      %r3, %r8                                                ;\
        EPILOGUE        
        
ENTRY_DIRECT(casueword32)
        CASUEWORD32(%r3, %r7)
END_DIRECT(casueword32)

#ifdef __powerpc64__
#define CASUEWORD64(raddr, rpcb)                                        ;\
        PROLOGUE                                                        ;\
        SET_FUSUFAULT(raddr, rpcb)                                      ;\
        li      %r8, 0                                                  ;\
1:                                                                      ;\
        ldarx   %r0, 0, %r3                                             ;\
        cmpld   %r4, %r0                                                ;\
        bne     2f                                                      ;\
        stdcx.  %r6, 0, %r3                                             ;\
        bne-    3f                                                      ;\
        b       4f                                                      ;\
2:                                                                      ;\
        stdcx.  %r0, 0, %r3             /* clear reservation (74xx) */  ;\
3:                                                                      ;\
        li      %r8, 1                                                  ;\
4:                                                                      ;\
        std     %r0, 0(%r5)                                             ;\
        CLEAR_FAULT_NO_CLOBBER(rpcb)                                    ;\
        mr      %r3, %r8                                                ;\
        EPILOGUE

ENTRY_DIRECT(casueword)
        CASUEWORD64(%r3, %r7)
END_DIRECT(casueword)

ENTRY_DIRECT(casueword64)
        CASUEWORD64(%r3, %r7)
END_DIRECT(casueword64)
#else
ENTRY_DIRECT(casueword)
        CASUEWORD32(%r3, %r7)
END_DIRECT(casueword)
#endif
        
_NAKED_ENTRY(fusufault)
        CLEAR_FAULT_NO_CLOBBER(%r7)
        li %r3, -1
        EPILOGUE
_END(fusufault)

_NAKED_ENTRY(copy_fault)
        CLEAR_FAULT_NO_CLOBBER(%r7)
        li %r3, EFAULT
        EPILOGUE
_END(copy_fault)