/*      $NetBSD: bus_space_asm_generic.S,v 1.7 2024/02/07 04:20:26 msaitoh Exp $        */

/*
 * Copyright (c) 2017 Ryo Shimizu
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <aarch64/asm.h>
#include "assym.h"

RCSID("$NetBSD: bus_space_asm_generic.S,v 1.7 2024/02/07 04:20:26 msaitoh Exp $")

.macro generate_bsfunc funcname, dsbop
/* uint8_t {funcname}_bs_r_1(x0:tag, x1:addr, x2:offset) */
ENTRY_NP(\funcname\()_bs_r_1)
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        ldrb    w0, [x1, x8]
        ret
END(\funcname\()_bs_r_1)


/* uint16_t {funcname}_bs_r_2(x0:tag, x1:addr, x2:offset) */
ENTRY_NP(\funcname\()_bs_r_2)
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        ldrh    w0, [x1, x8]
        ret
END(\funcname\()_bs_r_2)


/* uint32_t {funcname}_bs_r_4(x0:tag, x1:addr, x2:offset) */
ENTRY_NP(\funcname\()_bs_r_4)
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        ldr     w0, [x1, x8]
        ret
END(\funcname\()_bs_r_4)


/* uint64_t {funcname}_bs_r_8(x0:tag, x1:addr, x2:offset) */
ENTRY_NP(\funcname\()_bs_r_8)
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        ldr     x0, [x1, x8]
        ret
END(\funcname\()_bs_r_8)


/* uint16_t {funcname}_bs_r_2_swap(x0:tag, x1:addr, x2:offset) */
ENTRY_NP(\funcname\()_bs_r_2_swap)
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        ldrh    w0, [x1, x8]
        rev16   w0, w0
        ret
END(\funcname\()_bs_r_2_swap)


/* uint32_t {funcname}_bs_r_4_swap(x0:tag, x1:addr, x2:offset) */
ENTRY_NP(\funcname\()_bs_r_4_swap)
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        ldr     w0, [x1, x8]
        rev     w0, w0
        ret
END(\funcname\()_bs_r_4_swap)


/* uint64_t {funcname}_bs_r_8_swap(x0:tag, x1:addr, x2:offset) */
ENTRY_NP(\funcname\()_bs_r_8_swap)
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        ldr     x0, [x1, x8]
        rev     x0, x0
        ret
END(\funcname\()_bs_r_8_swap)


/* void {funcname}_bs_rm_1(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rm_1)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrb    w9, [x1, x8]
        subs    x4, x4, #1      /* count-- */
        strb    w9, [x3], #1
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rm_1: count == 0"
#endif
END(\funcname\()_bs_rm_1)


/* void {funcname}_bs_rm_2(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rm_2)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w9, [x1, x8]
        subs    x4, x4, #1      /* count-- */
        strh    w9, [x3], #2
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rm_2: count == 0"
#endif
END(\funcname\()_bs_rm_2)


/* void {funcname}_bs_rm_4(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rm_4)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w9, [x1, x8]
        subs    x4, x4, #1      /* count-- */
        str     w9, [x3], #4
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rm_4: count == 0"
#endif
END(\funcname\()_bs_rm_4)


/* void {funcname}_bs_rm_8(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rm_8)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x9, [x1, x8]
        subs    x4, x4, #1      /* count-- */
        str     x9, [x3], #8
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rm_8: count == 0"
#endif
END(\funcname\()_bs_rm_8)


/* void {funcname}_bs_rm_2_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rm_2_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w9, [x1, x8]
        subs    x4, x4, #1      /* count-- */
        rev16   w9, w9
        strh    w9, [x3], #2
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rm_2_swap: count == 0"
#endif
END(\funcname\()_bs_rm_2_swap)


/* void {funcname}_bs_rm_4_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rm_4_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w9, [x1, x8]
        subs    x4, x4, #1      /* count-- */
        rev     w9, w9
        str     w9, [x3], #4
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rm_4_swap: count == 0"
#endif
END(\funcname\()_bs_rm_4_swap)


/* void {funcname}_bs_rm_8_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rm_8_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x9, [x1, x8]
        subs    x4, x4, #1      /* count-- */
        rev     x9, x9
        str     x9, [x3], #8
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rm_8_swap: count == 0"
#endif
END(\funcname\()_bs_rm_8_swap)


/* void {funcname}_bs_rr_1(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rr_1)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrb    w8, [x1, x2]    /* value = *src */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* src += delta */
        strb    w8, [x3], #1    /* *dst++ = value */
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rr_1: count == 0"
#endif
END(\funcname\()_bs_rr_1)


/* void {funcname}_bs_rr_2(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rr_2)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #2          /* if (delta < 2) { */
        bcs     0f              /*   delta = 2; */
        mov     x9, #2          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w8, [x1, x2]    /* value = *src */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* src += delta */
        strh    w8, [x3], #2    /* *dst++ = value */
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rr_2: count == 0"
#endif
END(\funcname\()_bs_rr_2)


/* void {funcname}_bs_rr_4(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rr_4)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #4          /* if (delta < 4) { */
        bcs     0f              /*   delta = 4; */
        mov     x9, #4          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w8, [x1, x2]    /* value = *src */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* src += delta */
        str     w8, [x3], #4    /* *dst++ = value */
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rr_4: count == 0"
#endif
END(\funcname\()_bs_rr_4)


/* void {funcname}_bs_rr_8(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rr_8)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #8          /* if (delta < 8) { */
        bcs     0f              /*   delta = 8; */
        mov     x9, #8          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x8, [x1, x2]    /* value = *src */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* src += delta */
        str     x8, [x3], #8    /* *dst++ = value */
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rr_8: count == 0"
#endif
END(\funcname\()_bs_rr_8)


/* void {funcname}_bs_rr_2_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rr_2_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #2          /* if (delta < 2) { */
        bcs     0f              /*   delta = 2; */
        mov     x9, #2          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w8, [x1, x2]    /* value = *src */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* src += delta */
        rev16   w8, w8
        strh    w8, [x3], #2    /* *dst++ = value */
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rr_2_swap: count == 0"
#endif
END(\funcname\()_bs_rr_2_swap)


/* void {funcname}_bs_rr_4_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rr_4_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #4          /* if (delta < 4) { */
        bcs     0f              /*   delta = 4; */
        mov     x9, #4          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w8, [x1, x2]    /* value = *src */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* src += delta */
        rev     w8, w8
        str     w8, [x3], #4    /* *dst++ = value */
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rr_4_swap: count == 0"
#endif
END(\funcname\()_bs_rr_4_swap)


/* void {funcname}_bs_rr_8_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_rr_8_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        \dsbop
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #8          /* if (delta < 8) { */
        bcs     0f              /*   delta = 8; */
        mov     x9, #8          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x8, [x1, x2]    /* value = *src */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* src += delta */
        rev     x8, x8
        str     x8, [x3], #8    /* *dst++ = value */
        b.ne    1b
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_rr_8_swap: count == 0"
#endif
END(\funcname\()_bs_rr_8_swap)


/* void {funcname}_bs_sm_1(x0:tag, x1:addr, x2:offset, x3:data, x4:count) */
ENTRY_NP(\funcname\()_bs_sm_1)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        subs    x4, x4, #1      /* count-- */
        strb    w3, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sm_1: count == 0"
#endif
END(\funcname\()_bs_sm_1)


/* void {funcname}_bs_sm_2(x0:tag, x1:addr, x2:offset, x3:data, x4:count) */
/* void {funcname}_bs_sm_2_swap(x0:tag, x1:addr, x2:offset, x3:data, x4:count) */
ENTRY_NP(\funcname\()_bs_sm_2_swap)
        rev16   w3, w3
ENTRY_NP(\funcname\()_bs_sm_2)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        subs    x4, x4, #1      /* count-- */
        strh    w3, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sm_2(_swap): count == 0"
#endif
END(\funcname\()_bs_sm_2)
END(\funcname\()_bs_sm_2_swap)


/* void {funcname}_bs_sm_4(x0:tag, x1:addr, x2:offset, x3:data, x4:count) */
/* void {funcname}_bs_sm_4_swap(x0:tag, x1:addr, x2:offset, x3:data, x4:count) */
ENTRY_NP(\funcname\()_bs_sm_4_swap)
        rev     w3, w3
ENTRY_NP(\funcname\()_bs_sm_4)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        subs    x4, x4, #1      /* count-- */
        str     w3, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sm_4(_swap): count == 0"
#endif
END(\funcname\()_bs_sm_4)
END(\funcname\()_bs_sm_4_swap)


/* void {funcname}_bs_sm_8(x0:tag, x1:addr, x2:offset, x3:data, x4:count) */
/* void {funcname}_bs_sm_8_swap(x0:tag, x1:addr, x2:offset, x3:data, x4:count) */
ENTRY_NP(\funcname\()_bs_sm_8_swap)
        rev     x3, x3
ENTRY_NP(\funcname\()_bs_sm_8)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        subs    x4, x4, #1      /* count-- */
        str     x3, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sm_8(_swap): count == 0"
#endif
END(\funcname\()_bs_sm_8)
END(\funcname\()_bs_sm_8_swap)


/* void {funcname}_bs_w_1(x0:tag, x1:addr, x2:offset, x3:data) */
ENTRY_NP(\funcname\()_bs_w_1)
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        strb    w3, [x1, x8]
        \dsbop
        ret
END(\funcname\()_bs_w_1)


/* void {funcname}_bs_w_2(x0:tag, x1:addr, x2:offset, x3:data) */
/* void {funcname}_bs_w_2_swap(x0:tag, x1:addr, x2:offset, x3:data) */
ENTRY_NP(\funcname\()_bs_w_2_swap)
        rev16   w3, w3
ENTRY_NP(\funcname\()_bs_w_2)
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        strh    w3, [x1, x8]
        \dsbop
        ret
END(\funcname\()_bs_w_2)
END(\funcname\()_bs_w_2_swap)


/* void {funcname}_bs_w_4(x0:tag, x1:addr, x2:offset, x3:data) */
/* void {funcname}_bs_w_4_swap(x0:tag, x1:addr, x2:offset, x3:data) */
ENTRY_NP(\funcname\()_bs_w_4_swap)
        rev     w3, w3
ENTRY_NP(\funcname\()_bs_w_4)
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        str     w3, [x1, x8]
        \dsbop
        ret
END(\funcname\()_bs_w_4)
END(\funcname\()_bs_w_4_swap)


/* void {funcname}_bs_w_8(x0:tag, x1:addr, x2:offset, x3:data) */
/* void {funcname}_bs_w_8_swap(x0:tag, x1:addr, x2:offset, x3:data) */
ENTRY_NP(\funcname\()_bs_w_8_swap)
        rev     x3, x3
ENTRY_NP(\funcname\()_bs_w_8)
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
        str     x3, [x1, x8]
        \dsbop
        ret
END(\funcname\()_bs_w_8)
END(\funcname\()_bs_w_8_swap)


/* void {funcname}_bs_wm_1(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wm_1)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrb    w9, [x3], #1
        subs    x4, x4, #1      /* count-- */
        strb    w9, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wm_1: count == 0"
#endif
END(\funcname\()_bs_wm_1)


/* void {funcname}_bs_wm_2(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wm_2)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w9, [x3], #2
        subs    x4, x4, #1      /* count-- */
        strh    w9, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wm_2: count == 0"
#endif
END(\funcname\()_bs_wm_2)


/* void {funcname}_bs_wm_4(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wm_4)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w9, [x3], #4
        subs    x4, x4, #1      /* count-- */
        str     w9, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wm_4: count == 0"
#endif
END(\funcname\()_bs_wm_4)


/* void {funcname}_bs_wm_8(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wm_8)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x9, [x3], #8
        subs    x4, x4, #1      /* count-- */
        str     x9, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wm_8: count == 0"
#endif
END(\funcname\()_bs_wm_8)


/* void {funcname}_bs_wm_2_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wm_2_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w9, [x3], #2
        subs    x4, x4, #1      /* count-- */
        rev16   w9, w9
        strh    w9, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wm_2_swap: count == 0"
#endif
END(\funcname\()_bs_wm_2_swap)


/* void {funcname}_bs_wm_4_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wm_4_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w9, [x3], #4
        subs    x4, x4, #1      /* count-- */
        rev     w9, w9
        str     w9, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wm_4_swap: count == 0"
#endif
END(\funcname\()_bs_wm_4_swap)


/* void {funcname}_bs_wm_8_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wm_8_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x8, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x9, [x3], #8
        subs    x4, x4, #1      /* count-- */
        rev     x9, x9
        str     x9, [x1, x8]
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wm_8_swap: count == 0"
#endif
END(\funcname\()_bs_wm_8_swap)


/* void {funcname}_bs_wr_1(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wr_1)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrb    w8, [x3], #1    /* value = *src++ */
        subs    x4, x4, #1      /* count-- */
        strb    w8, [x1, x2]    /* *dst = value */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wr_1: count == 0"
#endif
END(\funcname\()_bs_wr_1)


/* void {funcname}_bs_wr_2(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wr_2)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #2          /* if (delta < 2) { */
        bcs     0f              /*   delta = 2; */
        mov     x9, #2          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w8, [x3], #2    /* value = *src++ */
        subs    x4, x4, #1      /* count-- */
        strh    w8, [x1, x2]    /* *dst = value */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wr_2: count == 0"
#endif
END(\funcname\()_bs_wr_2)


/* void {funcname}_bs_wr_4(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wr_4)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #4          /* if (delta < 4) { */
        bcs     0f              /*   delta = 4; */
        mov     x9, #4          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w8, [x3], #4    /* value = *src++ */
        subs    x4, x4, #1      /* count-- */
        str     w8, [x1, x2]    /* *dst = value */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wr_4: count == 0"
#endif
END(\funcname\()_bs_wr_4)


/* void {funcname}_bs_wr_8(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wr_8)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #8          /* if (delta < 8) { */
        bcs     0f              /*   delta = 8; */
        mov     x9, #8          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x8, [x3], #8    /* value = *src++ */
        subs    x4, x4, #1      /* count-- */
        str     x8, [x1, x2]    /* *dst = value */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wr_8: count == 0"
#endif
END(\funcname\()_bs_wr_8)


/* void {funcname}_bs_wr_2_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wr_2_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #2          /* if (delta < 2) { */
        bcs     0f              /*   delta = 2; */
        mov     x9, #2          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldrh    w8, [x3], #2    /* value = *src++ */
        rev16   w8, w8
        subs    x4, x4, #1      /* count-- */
        strh    w8, [x1, x2]    /* *dst = value */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wr_2_swap: count == 0"
#endif
END(\funcname\()_bs_wr_2_swap)


/* void {funcname}_bs_wr_4_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wr_4_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #4          /* if (delta < 4) { */
        bcs     0f              /*   delta = 4; */
        mov     x9, #4          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     w8, [x3], #4    /* value = *src++ */
        rev     w8, w8
        subs    x4, x4, #1      /* count-- */
        str     w8, [x1, x2]    /* *dst = value */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wr_4_swap: count == 0"
#endif
END(\funcname\()_bs_wr_4_swap)


/* void {funcname}_bs_wr_8_swap(x0:tag, x1:addr, x2:offset, x3:datap, x4:count) */
ENTRY_NP(\funcname\()_bs_wr_8_swap)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #8          /* if (delta < 8) { */
        bcs     0f              /*   delta = 8; */
        mov     x9, #8          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        ldr     x8, [x3], #8    /* value = *src++ */
        rev     x8, x8
        subs    x4, x4, #1      /* count-- */
        str     x8, [x1, x2]    /* *dst = value */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_wr_8_swap: count == 0"
#endif
END(\funcname\()_bs_wr_8_swap)


/* void {funcname}_bs_sr_1(x0:tag, x1:addr, x2:offset, x3:value, x4:count) */
ENTRY_NP(\funcname\()_bs_sr_1)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        strb    w3, [x1, x2]    /* *dst = value */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sr_1: count == 0"
#endif
END(\funcname\()_bs_sr_1)


/* void {funcname}_bs_sr_2(x0:tag, x1:addr, x2:offset, x3:value, x4:count) */
/* void {funcname}_bs_sr_2_swap(x0:tag, x1:addr, x2:offset, x3:value, x4:count) */
ENTRY_NP(\funcname\()_bs_sr_2_swap)
        rev16   w3, w3
ENTRY_NP(\funcname\()_bs_sr_2)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #2          /* if (delta < 2) { */
        bcs     0f              /*   delta = 2; */
        mov     x9, #2          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        strh    w3, [x1, x2]    /* *dst = value */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sr_2(_swap): count == 0"
#endif
END(\funcname\()_bs_sr_2)
END(\funcname\()_bs_sr_2_swap)


/* void {funcname}_bs_sr_4(x0:tag, x1:addr, x2:offset, x3:value, x4:count) */
/* void {funcname}_bs_sr_4_swap(x0:tag, x1:addr, x2:offset, x3:value, x4:count) */
ENTRY_NP(\funcname\()_bs_sr_4_swap)
        rev     w3, w3
ENTRY_NP(\funcname\()_bs_sr_4)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #4          /* if (delta < 4) { */
        bcs     0f              /*   delta = 4; */
        mov     x9, #4          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        str     w3, [x1, x2]    /* *dst = value */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sr_4(_swap): count == 0"
#endif
END(\funcname\()_bs_sr_4)
END(\funcname\()_bs_sr_4_swap)


/* void {funcname}_bs_sr_8(x0:tag, x1:addr, x2:offset, x3:value, x4:count) */
/* void {funcname}_bs_sr_8_swap(x0:tag, x1:addr, x2:offset, x3:value, x4:count) */
ENTRY_NP(\funcname\()_bs_sr_8_swap)
        rev     x3, x3
ENTRY_NP(\funcname\()_bs_sr_8)
#ifdef DIAGNOSTIC
        cbz     x4, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        mov     x9, #1
        lsl     x9, x9, x8      /* delta = 1 << tag->bs_stride */
        cmp     x9, #8          /* if (delta < 8) { */
        bcs     0f              /*   delta = 8; */
        mov     x9, #8          /* } */
0:
        lsl     x2, x2, x8      /* offset <<= tag->bs_stride */
1:
        str     x3, [x1, x2]    /* *dst = value */
        subs    x4, x4, #1      /* count-- */
        add     x2, x2, x9      /* dst += delta */
        b.ne    1b
        \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_sr_8(_swap): count == 0"
#endif
END(\funcname\()_bs_sr_8)
END(\funcname\()_bs_sr_8_swap)


/* void {funcname}_bs_c_1(x0:tag, x1:addr1, x2:offset1, x3:addr2, x4:offset2, x5:count) */
ENTRY_NP(\funcname\()_bs_c_1)
#ifdef DIAGNOSTIC
        cbz     x5, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x2, x2, x8      /* offset1 <<= tag->bs_stride */
        lsl     x4, x4, x8      /* offset2 <<= tag->bs_stride */
        add     x1, x1, x2      /* addr1 += offset1 */
        add     x3, x3, x4      /* addr2 += offset2 */
        mov     x10, #1
        lsl     x10, x10, x8    /* delta = (1 << tag->bs_stride) */
        cmp     x1, x3          /* addr1 > addr2 ? */
        b.cc    2f
1:
        /* in case of (addr1 > addr2) */
        ldrb    w9, [x1]
        subs    x5, x5, #1
        strb    w9, [x3]
        add     x1, x1, x10     /* addr1 += delta */
        add     x3, x3, x10     /* addr2 += delta */
        b.ne    1b
        b       8f
2:
        lsl     x8, x5, x8      /* (x8 = count * delta)  */
        add     x1, x1, x8      /* addr1 += count * delta */
        add     x3, x3, x8      /* addr2 += count * delta */
3:
        sub     x1, x1, x10     /* addr1 -= delta */
        sub     x3, x3, x10     /* addr2 -= delta */
        ldrb    w9, [x1]
        subs    x5, x5, #1
        strb    w9, [x3]
        b.ne    3b
8:      \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_c_1: count == 0"
#endif
END(\funcname\()_bs_c_1)


/* void {funcname}_bs_c_2(x0:tag, x1:addr1, x2:offset1, x3:addr2, x4:offset2, x5:count) */
ENTRY_NP(\funcname\()_bs_c_2)
#ifdef DIAGNOSTIC
        cbz     x5, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x2, x2, x8      /* offset1 <<= tag->bs_stride */
        lsl     x4, x4, x8      /* offset2 <<= tag->bs_stride */
        add     x1, x1, x2      /* addr1 += offset1 */
        add     x3, x3, x4      /* addr2 += offset2 */
        mov     x10, #1
        lsl     x10, x10, x8    /* delta = (1 << tag->bs_stride) */
        cmp     x10, #2         /* if (delta < 2) { */
        bcs     0f              /*   delta = 2; */
        mov     x10, #2         /* } */
0:
        cmp     x1, x3          /* addr1 > addr2 ? */
        b.cc    2f
1:
        /* in case of (addr1 > addr2) */
        ldrh    w9, [x1]
        subs    x5, x5, #1
        strh    w9, [x3]
        add     x1, x1, x10     /* addr1 += delta */
        add     x3, x3, x10     /* addr1 += delta */
        b.ne    1b
        b       8f
2:
        mul     x8, x5, x10     /* (x8 = count * delta) */
        add     x1, x1, x8      /* addr1 += count << tag->bs_stride */
        add     x3, x3, x8      /* addr2 += count << tag->bs_stride */
3:
        sub     x1, x1, x10     /* addr1 -= delta */
        sub     x3, x3, x10     /* addr1 -= delta */
        ldrh    w9, [x1]
        subs    x5, x5, #1
        strh    w9, [x3]
        b.ne    3b
8:      \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_c_2: count == 0"
#endif
END(\funcname\()_bs_c_2)


/* void {funcname}_bs_c_4(x0:tag, x1:addr1, x2:offset1, x3:addr2, x4:offset2, x5:count) */
ENTRY_NP(\funcname\()_bs_c_4)
#ifdef DIAGNOSTIC
        cbz     x5, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x2, x2, x8      /* offset1 <<= tag->bs_stride */
        lsl     x4, x4, x8      /* offset2 <<= tag->bs_stride */
        add     x1, x1, x2      /* addr1 += offset1 */
        add     x3, x3, x4      /* addr2 += offset2 */
        mov     x10, #1
        lsl     x10, x10, x8    /* delta = (1 << tag->bs_stride) */
        cmp     x10, #4         /* if (delta < 4) { */
        bcs     0f              /*   delta = 4; */
        mov     x10, #4         /* } */
0:
        cmp     x1, x3          /* addr1 > addr2 ? */
        b.cc    2f
1:
        /* in case of (addr1 > addr2) */
        ldr     w9, [x1]
        subs    x5, x5, #1
        str     w9, [x3]
        add     x1, x1, x10     /* addr1 += delta */
        add     x3, x3, x10     /* addr2 += delta */
        b.ne    1b
        b       8f
2:
        mul     x8, x5, x10     /* (x8 = count * delta) */
        add     x1, x1, x5      /* addr1 += count << tag->bs_delta */
        add     x3, x3, x5      /* addr2 += count << tag->bs_delta */
3:
        sub     x1, x1, x10     /* addr1 -= delta */
        sub     x3, x3, x10     /* addr1 -= delta */
        ldr     w9, [x1]
        subs    x5, x5, #1
        str     w9, [x3]
        b.ne    3b
8:      \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_c_4: count == 0"
#endif
END(\funcname\()_bs_c_4)


/* void {funcname}_bs_c_8(x0:tag, x1:addr1, x2:offset1, x3:addr2, x4:offset2, x5:count) */
ENTRY_NP(\funcname\()_bs_c_8)
#ifdef DIAGNOSTIC
        cbz     x5, 99f
#endif
        ldr     w8, [x0, #BS_STRIDE]
        lsl     x2, x2, x8      /* offset1 <<= tag->bs_stride */
        lsl     x4, x4, x8      /* offset2 <<= tag->bs_stride */
        add     x1, x1, x2      /* addr1 += offset1 */
        add     x3, x3, x4      /* addr2 += offset2 */
        mov     x10, #1
        lsl     x10, x10, x8    /* delta = (1 << tag->bs_stride) */
        cmp     x10, #8         /* if (delta < 8) { */
        bcs     0f              /*   delta = 8; */
        mov     x10, #8         /* } */
0:
        cmp     x1, x3          /* addr1 > addr2 ? */
        b.cc    2f
1:
        /* in case of (addr1 > addr2) */
        ldr     x9, [x1]
        subs    x5, x5, #1
        str     x9, [x3]
        add     x1, x1, x10     /* addr1 += delta */
        add     x3, x3, x10     /* addr1 += delta */
        b.ne    1b
        b       8f
2:
        mul     x8, x5, x10     /* (x8 = count * delta) */
        add     x1, x1, x8      /* addr1 += count << tag->bs_stride */
        add     x3, x3, x8      /* addr2 += count << tag->bs_stride */
3:
        sub     x1, x1, x10     /* addr1 -= delta */
        sub     x3, x3, x10     /* addr1 -= delta */
        ldr     x9, [x1]
        subs    x5, x5, #1
        str     x9, [x3]
        b.ne    3b
8:      \dsbop
        ret
#ifdef DIAGNOSTIC
99:     adr     x0, 100f
        b       _C_LABEL(panic)
100:    .asciz  "\funcname\()_bs_c_8: count == 0"
#endif
END(\funcname\()_bs_c_8)
.endm


generate_bsfunc generic
generate_bsfunc generic_dsb, "dsb sy"