// SPDX-License-Identifier: GPL-2.0-only
// Copyright (C) 2021 ARM Limited.
//
// Assembly portion of the syscall ABI test

//
// Load values from memory into registers, invoke a syscall and save the
// register values back to memory for later checking.  The syscall to be
// invoked is configured in x8 of the input GPR data.
//
// x0:  SVE VL, 0 for FP only
// x1:  SME VL
//
//      GPRs:   gpr_in, gpr_out
//      FPRs:   fpr_in, fpr_out
//      Zn:     z_in, z_out
//      Pn:     p_in, p_out
//      FFR:    ffr_in, ffr_out
//      ZA:     za_in, za_out
//      SVCR:   svcr_in, svcr_out

#include "syscall-abi.h"

.arch_extension sve

#define ID_AA64SMFR0_EL1_SMEver_SHIFT           56
#define ID_AA64SMFR0_EL1_SMEver_WIDTH           4

/*
 * LDR (vector to ZA array):
 *      LDR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL]
 */
.macro _ldr_za nw, nxbase, offset=0
        .inst   0xe1000000                      \
                | (((\nw) & 3) << 13)           \
                | ((\nxbase) << 5)              \
                | ((\offset) & 7)
.endm

/*
 * STR (vector from ZA array):
 *      STR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL]
 */
.macro _str_za nw, nxbase, offset=0
        .inst   0xe1200000                      \
                | (((\nw) & 3) << 13)           \
                | ((\nxbase) << 5)              \
                | ((\offset) & 7)
.endm

/*
 * LDR (ZT0)
 *
 *      LDR ZT0, nx
 */
.macro _ldr_zt nx
        .inst   0xe11f8000                      \
                | (((\nx) & 0x1f) << 5)
.endm

/*
 * STR (ZT0)
 *
 *      STR ZT0, nx
 */
.macro _str_zt nx
        .inst   0xe13f8000                      \
                | (((\nx) & 0x1f) << 5)
.endm

.globl do_syscall
do_syscall:
        // Store callee saved registers x19-x29 (80 bytes) plus x0 and x1
        stp     x29, x30, [sp, #-112]!
        mov     x29, sp
        stp     x0, x1, [sp, #16]
        stp     x19, x20, [sp, #32]
        stp     x21, x22, [sp, #48]
        stp     x23, x24, [sp, #64]
        stp     x25, x26, [sp, #80]
        stp     x27, x28, [sp, #96]

        // Set SVCR if we're doing SME
        cbz     x1, load_gpr
        adrp    x2, svcr_in
        ldr     x2, [x2, :lo12:svcr_in]
        msr     S3_3_C4_C2_2, x2

        // Load ZA and ZT0 if enabled - uses x12 as scratch due to SME LDR
        tbz     x2, #SVCR_ZA_SHIFT, load_gpr
        mov     w12, #0
        ldr     x2, =za_in
1:      _ldr_za 12, 2
        add     x2, x2, x1
        add     x12, x12, #1
        cmp     x1, x12
        bne     1b

        // ZT0
        mrs     x2, S3_0_C0_C4_5        // ID_AA64SMFR0_EL1
        ubfx    x2, x2, #ID_AA64SMFR0_EL1_SMEver_SHIFT, \
                         #ID_AA64SMFR0_EL1_SMEver_WIDTH
        cbz     x2, load_gpr
        adrp    x2, zt_in
        add     x2, x2, :lo12:zt_in
        _ldr_zt 2

load_gpr:
        // Load GPRs x8-x28, and save our SP/FP for later comparison
        ldr     x2, =gpr_in
        add     x2, x2, #64
        ldp     x8, x9, [x2], #16
        ldp     x10, x11, [x2], #16
        ldp     x12, x13, [x2], #16
        ldp     x14, x15, [x2], #16
        ldp     x16, x17, [x2], #16
        ldp     x18, x19, [x2], #16
        ldp     x20, x21, [x2], #16
        ldp     x22, x23, [x2], #16
        ldp     x24, x25, [x2], #16
        ldp     x26, x27, [x2], #16
        ldr     x28, [x2], #8
        str     x29, [x2], #8           // FP
        str     x30, [x2], #8           // LR

        // Load FPRs if we're not doing neither SVE nor streaming SVE
        cbnz    x0, check_sve_in
        ldr     x2, =svcr_in
        tbnz    x2, #SVCR_SM_SHIFT, check_sve_in

        ldr     x2, =fpr_in
        ldp     q0, q1, [x2]
        ldp     q2, q3, [x2, #16 * 2]
        ldp     q4, q5, [x2, #16 * 4]
        ldp     q6, q7, [x2, #16 * 6]
        ldp     q8, q9, [x2, #16 * 8]
        ldp     q10, q11, [x2, #16 * 10]
        ldp     q12, q13, [x2, #16 * 12]
        ldp     q14, q15, [x2, #16 * 14]
        ldp     q16, q17, [x2, #16 * 16]
        ldp     q18, q19, [x2, #16 * 18]
        ldp     q20, q21, [x2, #16 * 20]
        ldp     q22, q23, [x2, #16 * 22]
        ldp     q24, q25, [x2, #16 * 24]
        ldp     q26, q27, [x2, #16 * 26]
        ldp     q28, q29, [x2, #16 * 28]
        ldp     q30, q31, [x2, #16 * 30]

        b       2f

check_sve_in:
        // Load the SVE registers if we're doing SVE/SME

        ldr     x2, =z_in
        ldr     z0, [x2, #0, MUL VL]
        ldr     z1, [x2, #1, MUL VL]
        ldr     z2, [x2, #2, MUL VL]
        ldr     z3, [x2, #3, MUL VL]
        ldr     z4, [x2, #4, MUL VL]
        ldr     z5, [x2, #5, MUL VL]
        ldr     z6, [x2, #6, MUL VL]
        ldr     z7, [x2, #7, MUL VL]
        ldr     z8, [x2, #8, MUL VL]
        ldr     z9, [x2, #9, MUL VL]
        ldr     z10, [x2, #10, MUL VL]
        ldr     z11, [x2, #11, MUL VL]
        ldr     z12, [x2, #12, MUL VL]
        ldr     z13, [x2, #13, MUL VL]
        ldr     z14, [x2, #14, MUL VL]
        ldr     z15, [x2, #15, MUL VL]
        ldr     z16, [x2, #16, MUL VL]
        ldr     z17, [x2, #17, MUL VL]
        ldr     z18, [x2, #18, MUL VL]
        ldr     z19, [x2, #19, MUL VL]
        ldr     z20, [x2, #20, MUL VL]
        ldr     z21, [x2, #21, MUL VL]
        ldr     z22, [x2, #22, MUL VL]
        ldr     z23, [x2, #23, MUL VL]
        ldr     z24, [x2, #24, MUL VL]
        ldr     z25, [x2, #25, MUL VL]
        ldr     z26, [x2, #26, MUL VL]
        ldr     z27, [x2, #27, MUL VL]
        ldr     z28, [x2, #28, MUL VL]
        ldr     z29, [x2, #29, MUL VL]
        ldr     z30, [x2, #30, MUL VL]
        ldr     z31, [x2, #31, MUL VL]

        // Only set a non-zero FFR, test patterns must be zero since the
        // syscall should clear it - this lets us handle FA64.
        ldr     x2, =ffr_in
        ldr     p0, [x2]
        ldr     x2, [x2, #0]
        cbz     x2, 1f
        wrffr   p0.b
1:

        ldr     x2, =p_in
        ldr     p0, [x2, #0, MUL VL]
        ldr     p1, [x2, #1, MUL VL]
        ldr     p2, [x2, #2, MUL VL]
        ldr     p3, [x2, #3, MUL VL]
        ldr     p4, [x2, #4, MUL VL]
        ldr     p5, [x2, #5, MUL VL]
        ldr     p6, [x2, #6, MUL VL]
        ldr     p7, [x2, #7, MUL VL]
        ldr     p8, [x2, #8, MUL VL]
        ldr     p9, [x2, #9, MUL VL]
        ldr     p10, [x2, #10, MUL VL]
        ldr     p11, [x2, #11, MUL VL]
        ldr     p12, [x2, #12, MUL VL]
        ldr     p13, [x2, #13, MUL VL]
        ldr     p14, [x2, #14, MUL VL]
        ldr     p15, [x2, #15, MUL VL]
2:

        // Do the syscall
        svc     #0

        // Save GPRs x8-x30
        ldr     x2, =gpr_out
        add     x2, x2, #64
        stp     x8, x9, [x2], #16
        stp     x10, x11, [x2], #16
        stp     x12, x13, [x2], #16
        stp     x14, x15, [x2], #16
        stp     x16, x17, [x2], #16
        stp     x18, x19, [x2], #16
        stp     x20, x21, [x2], #16
        stp     x22, x23, [x2], #16
        stp     x24, x25, [x2], #16
        stp     x26, x27, [x2], #16
        stp     x28, x29, [x2], #16
        str     x30, [x2]

        // Restore x0 and x1 for feature checks
        ldp     x0, x1, [sp, #16]

        // Save FPSIMD state
        ldr     x2, =fpr_out
        stp     q0, q1, [x2]
        stp     q2, q3, [x2, #16 * 2]
        stp     q4, q5, [x2, #16 * 4]
        stp     q6, q7, [x2, #16 * 6]
        stp     q8, q9, [x2, #16 * 8]
        stp     q10, q11, [x2, #16 * 10]
        stp     q12, q13, [x2, #16 * 12]
        stp     q14, q15, [x2, #16 * 14]
        stp     q16, q17, [x2, #16 * 16]
        stp     q18, q19, [x2, #16 * 18]
        stp     q20, q21, [x2, #16 * 20]
        stp     q22, q23, [x2, #16 * 22]
        stp     q24, q25, [x2, #16 * 24]
        stp     q26, q27, [x2, #16 * 26]
        stp     q28, q29, [x2, #16 * 28]
        stp     q30, q31, [x2, #16 * 30]

        // Save SVCR if we're doing SME
        cbz     x1, check_sve_out
        mrs     x2, S3_3_C4_C2_2
        adrp    x3, svcr_out
        str     x2, [x3, :lo12:svcr_out]

        // Save ZA if it's enabled - uses x12 as scratch due to SME STR
        tbz     x2, #SVCR_ZA_SHIFT, check_sve_out
        mov     w12, #0
        ldr     x2, =za_out
1:      _str_za 12, 2
        add     x2, x2, x1
        add     x12, x12, #1
        cmp     x1, x12
        bne     1b

        // ZT0
        mrs     x2, S3_0_C0_C4_5        // ID_AA64SMFR0_EL1
        ubfx    x2, x2, #ID_AA64SMFR0_EL1_SMEver_SHIFT, \
                        #ID_AA64SMFR0_EL1_SMEver_WIDTH
        cbz     x2, check_sve_out
        adrp    x2, zt_out
        add     x2, x2, :lo12:zt_out
        _str_zt 2

check_sve_out:
        // Save the SVE state if we have some
        cbz     x0, 1f

        ldr     x2, =z_out
        str     z0, [x2, #0, MUL VL]
        str     z1, [x2, #1, MUL VL]
        str     z2, [x2, #2, MUL VL]
        str     z3, [x2, #3, MUL VL]
        str     z4, [x2, #4, MUL VL]
        str     z5, [x2, #5, MUL VL]
        str     z6, [x2, #6, MUL VL]
        str     z7, [x2, #7, MUL VL]
        str     z8, [x2, #8, MUL VL]
        str     z9, [x2, #9, MUL VL]
        str     z10, [x2, #10, MUL VL]
        str     z11, [x2, #11, MUL VL]
        str     z12, [x2, #12, MUL VL]
        str     z13, [x2, #13, MUL VL]
        str     z14, [x2, #14, MUL VL]
        str     z15, [x2, #15, MUL VL]
        str     z16, [x2, #16, MUL VL]
        str     z17, [x2, #17, MUL VL]
        str     z18, [x2, #18, MUL VL]
        str     z19, [x2, #19, MUL VL]
        str     z20, [x2, #20, MUL VL]
        str     z21, [x2, #21, MUL VL]
        str     z22, [x2, #22, MUL VL]
        str     z23, [x2, #23, MUL VL]
        str     z24, [x2, #24, MUL VL]
        str     z25, [x2, #25, MUL VL]
        str     z26, [x2, #26, MUL VL]
        str     z27, [x2, #27, MUL VL]
        str     z28, [x2, #28, MUL VL]
        str     z29, [x2, #29, MUL VL]
        str     z30, [x2, #30, MUL VL]
        str     z31, [x2, #31, MUL VL]

        ldr     x2, =p_out
        str     p0, [x2, #0, MUL VL]
        str     p1, [x2, #1, MUL VL]
        str     p2, [x2, #2, MUL VL]
        str     p3, [x2, #3, MUL VL]
        str     p4, [x2, #4, MUL VL]
        str     p5, [x2, #5, MUL VL]
        str     p6, [x2, #6, MUL VL]
        str     p7, [x2, #7, MUL VL]
        str     p8, [x2, #8, MUL VL]
        str     p9, [x2, #9, MUL VL]
        str     p10, [x2, #10, MUL VL]
        str     p11, [x2, #11, MUL VL]
        str     p12, [x2, #12, MUL VL]
        str     p13, [x2, #13, MUL VL]
        str     p14, [x2, #14, MUL VL]
        str     p15, [x2, #15, MUL VL]

        // Only save FFR if we wrote a value for SME
        ldr     x2, =ffr_in
        ldr     x2, [x2, #0]
        cbz     x2, 1f
        ldr     x2, =ffr_out
        rdffr   p0.b
        str     p0, [x2]
1:

        // Restore callee saved registers x19-x30
        ldp     x19, x20, [sp, #32]
        ldp     x21, x22, [sp, #48]
        ldp     x23, x24, [sp, #64]
        ldp     x25, x26, [sp, #80]
        ldp     x27, x28, [sp, #96]
        ldp     x29, x30, [sp], #112

        // Clear SVCR if we were doing SME so future tests don't have ZA
        cbz     x1, 1f
        msr     S3_3_C4_C2_2, xzr
1:

        ret