/*
 * M7memcpy: Optimized SPARC M7 memcpy
 *
 * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
 */

        .file   "M7memcpy.S"

/*
 * memcpy(s1, s2, len)
 *
 * Copy s2 to s1, always copy n bytes.
 * Note: this C code does not work for overlapped copies.
 *
 * Fast assembler language version of the following C-program for memcpy
 * which represents the `standard' for the C-library.
 *
 *      void *
 *      memcpy(void *s, const void *s0, size_t n)
 *      {
 *              if (n != 0) {
 *                  char *s1 = s;
 *                  const char *s2 = s0;
 *                  do {
 *                      *s1++ = *s2++;
 *                  } while (--n != 0);
 *              }
 *              return (s);
 *      }
 *
 *
 * SPARC T7/M7 Flow :
 *
 * if (count < SMALL_MAX) {
 *   if count < SHORTCOPY              (SHORTCOPY=3)
 *      copy bytes; exit with dst addr
 *   if src & dst aligned on word boundary but not long word boundary,
 *     copy with ldw/stw; branch to finish_up
 *   if src & dst aligned on long word boundary
 *     copy with ldx/stx; branch to finish_up
 *   if src & dst not aligned and length <= SHORTCHECK   (SHORTCHECK=14)
 *     copy bytes; exit with dst addr
 *   move enough bytes to get src to word boundary
 *   if dst now on word boundary
 * move_words:
 *     copy words; branch to finish_up
 *   if dst now on half word boundary
 *     load words, shift half words, store words; branch to finish_up
 *   if dst on byte 1
 *     load words, shift 3 bytes, store words; branch to finish_up
 *   if dst on byte 3
 *     load words, shift 1 byte, store words; branch to finish_up
 * finish_up:
 *     copy bytes; exit with dst addr
 * } else {                                         More than SMALL_MAX bytes
 *   move bytes until dst is on long word boundary
 *   if( src is on long word boundary ) {
 *     if (count < MED_MAX) {
 * finish_long:                                    src/dst aligned on 8 bytes
 *       copy with ldx/stx in 8-way unrolled loop;
 *       copy final 0-63 bytes; exit with dst addr
 *     } else {                              src/dst aligned; count > MED_MAX
 *       align dst on 64 byte boundary; for main data movement:
 *       prefetch src data to L2 cache; let HW prefetch move data to L1 cache
 *       Use BIS (block initializing store) to avoid copying store cache
 *       lines from memory. But pre-store first element of each cache line
 *       ST_CHUNK lines in advance of the rest of that cache line. That
 *       gives time for replacement cache lines to be written back without
 *       excess STQ and Miss Buffer filling. Repeat until near the end,
 *       then finish up storing before going to finish_long.
 *     }
 *   } else {                                   src/dst not aligned on 8 bytes
 *     if src is word aligned and count < MED_WMAX
 *       move words in 8-way unrolled loop
 *       move final 0-31 bytes; exit with dst addr
 *     if count < MED_UMAX
 *       use alignaddr/faligndata combined with ldd/std in 8-way
 *       unrolled loop to move data.
 *       go to unalign_done
 *     else
 *       setup alignaddr for faligndata instructions
 *       align dst on 64 byte boundary; prefetch src data to L1 cache
 *       loadx8, falign, block-store, prefetch loop
 *       (only use block-init-store when src/dst on 8 byte boundaries.)
 * unalign_done:
 *       move remaining bytes for unaligned cases. exit with dst addr.
 * }
 *
 */

#include <asm/visasm.h>
#include <asm/asi.h>

#if !defined(EX_LD) && !defined(EX_ST)
#define NON_USER_COPY
#endif

#ifndef EX_LD
#define EX_LD(x,y)      x
#endif
#ifndef EX_LD_FP
#define EX_LD_FP(x,y)   x
#endif

#ifndef EX_ST
#define EX_ST(x,y)      x
#endif
#ifndef EX_ST_FP
#define EX_ST_FP(x,y)   x
#endif

#ifndef EX_RETVAL
#define EX_RETVAL(x)    x
#endif

#ifndef LOAD
#define LOAD(type,addr,dest)    type [addr], dest
#endif

#ifndef STORE
#define STORE(type,src,addr)    type src, [addr]
#endif

/*
 * ASI_BLK_INIT_QUAD_LDD_P/ASI_BLK_INIT_QUAD_LDD_S marks the cache
 * line as "least recently used" which means if many threads are
 * active, it has a high probability of being pushed out of the cache
 * between the first initializing store and the final stores.
 * Thus, we use ASI_ST_BLKINIT_MRU_P/ASI_ST_BLKINIT_MRU_S which
 * marks the cache line as "most recently used" for all
 * but the last cache line
 */
#ifndef STORE_ASI
#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
#define STORE_ASI       ASI_BLK_INIT_QUAD_LDD_P
#else
#define STORE_ASI       0x80            /* ASI_P */
#endif
#endif

#ifndef STORE_MRU_ASI
#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
#define STORE_MRU_ASI   ASI_ST_BLKINIT_MRU_P
#else
#define STORE_MRU_ASI   0x80            /* ASI_P */
#endif
#endif

#ifndef STORE_INIT
#define STORE_INIT(src,addr)    stxa src, [addr] STORE_ASI
#endif

#ifndef STORE_INIT_MRU
#define STORE_INIT_MRU(src,addr)        stxa src, [addr] STORE_MRU_ASI
#endif

#ifndef FUNC_NAME
#define FUNC_NAME       M7memcpy
#endif

#ifndef PREAMBLE
#define PREAMBLE
#endif

#define BLOCK_SIZE      64
#define SHORTCOPY       3
#define SHORTCHECK      14
#define SHORT_LONG      64      /* max copy for short longword-aligned case */
                                /* must be at least 64 */
#define SMALL_MAX       128
#define MED_UMAX        1024    /* max copy for medium un-aligned case */
#define MED_WMAX        1024    /* max copy for medium word-aligned case */
#define MED_MAX         1024    /* max copy for medium longword-aligned case */
#define ST_CHUNK        24      /* ST_CHUNK - block of values for BIS Store */
#define ALIGN_PRE       24      /* distance for aligned prefetch loop */

        .register       %g2,#scratch

        .section        ".text"
        .global         FUNC_NAME
        .type           FUNC_NAME, #function
        .align          16
FUNC_NAME:
        srlx            %o2, 31, %g2
        cmp             %g2, 0
        tne             %xcc, 5
        PREAMBLE
        mov             %o0, %g1        ! save %o0
        brz,pn          %o2, .Lsmallx
         cmp            %o2, 3
        ble,pn          %icc, .Ltiny_cp
         cmp            %o2, 19
        ble,pn          %icc, .Lsmall_cp
         or             %o0, %o1, %g2
        cmp             %o2, SMALL_MAX
        bl,pn           %icc, .Lmedium_cp
         nop

.Lmedium:
        neg     %o0, %o5
        andcc   %o5, 7, %o5             ! bytes till DST 8 byte aligned
        brz,pt  %o5, .Ldst_aligned_on_8

        ! %o5 has the bytes to be written in partial store.
         sub    %o2, %o5, %o2
        sub     %o1, %o0, %o1           ! %o1 gets the difference
7:                                      ! dst aligning loop
        add     %o1, %o0, %o4
        EX_LD(LOAD(ldub, %o4, %o4), memcpy_retl_o2_plus_o5)     ! load one byte
        subcc   %o5, 1, %o5
        EX_ST(STORE(stb, %o4, %o0), memcpy_retl_o2_plus_o5_plus_1)
        bgu,pt  %xcc, 7b
         add    %o0, 1, %o0             ! advance dst
        add     %o1, %o0, %o1           ! restore %o1
.Ldst_aligned_on_8:
        andcc   %o1, 7, %o5
        brnz,pt %o5, .Lsrc_dst_unaligned_on_8
         nop

.Lsrc_dst_aligned_on_8:
        ! check if we are copying MED_MAX or more bytes
        set MED_MAX, %o3
        cmp %o2, %o3                    ! limit to store buffer size
        bgu,pn  %xcc, .Llarge_align8_copy
         nop

/*
 * Special case for handling when src and dest are both long word aligned
 * and total data to move is less than MED_MAX bytes
 */
.Lmedlong:
        subcc   %o2, 63, %o2            ! adjust length to allow cc test
        ble,pn  %xcc, .Lmedl63          ! skip big loop if less than 64 bytes
         nop
.Lmedl64:
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_63)      ! load
        subcc   %o2, 64, %o2            ! decrement length count
        EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_63_64)  ! and store
        EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_63_56) ! a block of 64
        EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_63_56)
        EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_63_48)
        EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_63_48)
        EX_LD(LOAD(ldx, %o1+24, %o3), memcpy_retl_o2_plus_63_40)
        EX_ST(STORE(stx, %o3, %o0+24), memcpy_retl_o2_plus_63_40)
        EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_63_32)! load and store
        EX_ST(STORE(stx, %o4, %o0+32), memcpy_retl_o2_plus_63_32)
        EX_LD(LOAD(ldx, %o1+40, %o3), memcpy_retl_o2_plus_63_24)! a block of 64
        add     %o1, 64, %o1            ! increase src ptr by 64
        EX_ST(STORE(stx, %o3, %o0+40), memcpy_retl_o2_plus_63_24)
        EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_63_16)
        add     %o0, 64, %o0            ! increase dst ptr by 64
        EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_63_16)
        EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_63_8)
        bgu,pt  %xcc, .Lmedl64          ! repeat if at least 64 bytes left
         EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_63_8)
.Lmedl63:
        addcc   %o2, 32, %o2            ! adjust remaining count
        ble,pt  %xcc, .Lmedl31          ! to skip if 31 or fewer bytes left
         nop
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_31)      ! load
        sub     %o2, 32, %o2            ! decrement length count
        EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_31_32)  ! and store
        EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_31_24) ! a block of 32
        add     %o1, 32, %o1            ! increase src ptr by 32
        EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_31_24)
        EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_31_16)
        add     %o0, 32, %o0            ! increase dst ptr by 32
        EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_31_16)
        EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_31_8)
        EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_31_8)
.Lmedl31:
        addcc   %o2, 16, %o2            ! adjust remaining count
        ble,pt  %xcc, .Lmedl15          ! skip if 15 or fewer bytes left
         nop                            !
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_15)
        add     %o1, 16, %o1            ! increase src ptr by 16
        EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_15)
        sub     %o2, 16, %o2            ! decrease count by 16
        EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_15_8)
        add     %o0, 16, %o0            ! increase dst ptr by 16
        EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_15_8)
.Lmedl15:
        addcc   %o2, 15, %o2            ! restore count
        bz,pt   %xcc, .Lsmallx  ! exit if finished
         cmp    %o2, 8
        blt,pt  %xcc, .Lmedw7           ! skip if 7 or fewer bytes left
         tst    %o2
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2)      ! load 8 bytes
        add     %o1, 8, %o1             ! increase src ptr by 8
        add     %o0, 8, %o0             ! increase dst ptr by 8
        subcc   %o2, 8, %o2             ! decrease count by 8
        bnz,pn  %xcc, .Lmedw7
         EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8)   ! and store 8
        retl
         mov    EX_RETVAL(%g1), %o0     ! restore %o0

        .align 16
.Lsrc_dst_unaligned_on_8:
        ! DST is 8-byte aligned, src is not
2:
        andcc   %o1, 0x3, %o5           ! test word alignment
        bnz,pt  %xcc, .Lunalignsetup    ! branch to skip if not word aligned
         nop

/*
 * Handle all cases where src and dest are aligned on word
 * boundaries. Use unrolled loops for better performance.
 * This option wins over standard large data move when
 * source and destination is in cache for.Lmedium
 * to short data moves.
 */
        set MED_WMAX, %o3
        cmp %o2, %o3                    ! limit to store buffer size
        bge,pt  %xcc, .Lunalignrejoin   ! otherwise rejoin main loop
         nop

        subcc   %o2, 31, %o2            ! adjust length to allow cc test
                                        ! for end of loop
        ble,pt  %xcc, .Lmedw31          ! skip big loop if less than 16
.Lmedw32:
        EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_31)! move a block of 32
        sllx    %o4, 32, %o5
        EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_31)
        or      %o4, %o5, %o5
        EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_31)
        subcc   %o2, 32, %o2            ! decrement length count
        EX_LD(LOAD(ld, %o1+8, %o4), memcpy_retl_o2_plus_31_24)
        sllx    %o4, 32, %o5
        EX_LD(LOAD(ld, %o1+12, %o4), memcpy_retl_o2_plus_31_24)
        or      %o4, %o5, %o5
        EX_ST(STORE(stx, %o5, %o0+8), memcpy_retl_o2_plus_31_24)
        add     %o1, 32, %o1            ! increase src ptr by 32
        EX_LD(LOAD(ld, %o1-16, %o4), memcpy_retl_o2_plus_31_16)
        sllx    %o4, 32, %o5
        EX_LD(LOAD(ld, %o1-12, %o4), memcpy_retl_o2_plus_31_16)
        or      %o4, %o5, %o5
        EX_ST(STORE(stx, %o5, %o0+16), memcpy_retl_o2_plus_31_16)
        add     %o0, 32, %o0            ! increase dst ptr by 32
        EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_31_8)
        sllx    %o4, 32, %o5
        EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_31_8)
        or      %o4, %o5, %o5
        bgu,pt  %xcc, .Lmedw32          ! repeat if at least 32 bytes left
         EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_31_8)
.Lmedw31:
        addcc   %o2, 31, %o2            ! restore count

        bz,pt   %xcc, .Lsmallx  ! exit if finished
         nop
        cmp     %o2, 16
        blt,pt  %xcc, .Lmedw15
         nop
        EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2)! move a block of 16 bytes
        sllx    %o4, 32, %o5
        subcc   %o2, 16, %o2            ! decrement length count
        EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_16)
        or      %o4, %o5, %o5
        EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_16)
        add     %o1, 16, %o1            ! increase src ptr by 16
        EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_8)
        add     %o0, 16, %o0            ! increase dst ptr by 16
        sllx    %o4, 32, %o5
        EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_8)
        or      %o4, %o5, %o5
        EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_8)
.Lmedw15:
        bz,pt   %xcc, .Lsmallx  ! exit if finished
         cmp    %o2, 8
        blt,pn  %xcc, .Lmedw7           ! skip if 7 or fewer bytes left
         tst    %o2
        EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2)       ! load 4 bytes
        subcc   %o2, 8, %o2             ! decrease count by 8
        EX_ST(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_8)! and store 4 bytes
        add     %o1, 8, %o1             ! increase src ptr by 8
        EX_LD(LOAD(ld, %o1-4, %o3), memcpy_retl_o2_plus_4)      ! load 4 bytes
        add     %o0, 8, %o0             ! increase dst ptr by 8
        EX_ST(STORE(stw, %o3, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes
        bz,pt   %xcc, .Lsmallx  ! exit if finished
.Lmedw7:                                ! count is ge 1, less than 8
        cmp     %o2, 4                  ! check for 4 bytes left
        blt,pn  %xcc, .Lsmallleft3      ! skip if 3 or fewer bytes left
         nop                            !
        EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2)       ! load 4 bytes
        add     %o1, 4, %o1             ! increase src ptr by 4
        add     %o0, 4, %o0             ! increase dst ptr by 4
        subcc   %o2, 4, %o2             ! decrease count by 4
        bnz     .Lsmallleft3
         EX_ST(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes
        retl
         mov    EX_RETVAL(%g1), %o0

        .align 16
.Llarge_align8_copy:                    ! Src and dst share 8 byte alignment
        ! align dst to 64 byte boundary
        andcc   %o0, 0x3f, %o3          ! %o3 == 0 means dst is 64 byte aligned
        brz,pn  %o3, .Laligned_to_64
         andcc  %o0, 8, %o3             ! odd long words to move?
        brz,pt  %o3, .Laligned_to_16
         nop
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2)
        sub     %o2, 8, %o2
        add     %o1, 8, %o1             ! increment src ptr
        add     %o0, 8, %o0             ! increment dst ptr
        EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8)
.Laligned_to_16:
        andcc   %o0, 16, %o3            ! pair of long words to move?
        brz,pt  %o3, .Laligned_to_32
         nop
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2)
        sub     %o2, 16, %o2
        EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_16)
        add     %o1, 16, %o1            ! increment src ptr
        EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8)
        add     %o0, 16, %o0            ! increment dst ptr
        EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8)
.Laligned_to_32:
        andcc   %o0, 32, %o3            ! four long words to move?
        brz,pt  %o3, .Laligned_to_64
         nop
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2)
        sub     %o2, 32, %o2
        EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_32)
        EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_24)
        EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_24)
        EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_16)
        EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_16)
        add     %o1, 32, %o1            ! increment src ptr
        EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8)
        add     %o0, 32, %o0            ! increment dst ptr
        EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8)
.Laligned_to_64:
!
!       Using block init store (BIS) instructions to avoid fetching cache
!       lines from memory. Use ST_CHUNK stores to first element of each cache
!       line (similar to prefetching) to avoid overfilling STQ or miss buffers.
!       Gives existing cache lines time to be moved out of L1/L2/L3 cache.
!       Initial stores using MRU version of BIS to keep cache line in
!       cache until we are ready to store final element of cache line.
!       Then store last element using the LRU version of BIS.
!
        andn    %o2, 0x3f, %o5          ! %o5 is multiple of block size
        and     %o2, 0x3f, %o2          ! residue bytes in %o2
!
!       We use STORE_MRU_ASI for the first seven stores to each cache line
!       followed by STORE_ASI (mark as LRU) for the last store. That
!       mixed approach reduces the probability that the cache line is removed
!       before we finish setting it, while minimizing the effects on
!       other cached values during a large memcpy
!
!       ST_CHUNK batches up initial BIS operations for several cache lines
!       to allow multiple requests to not be blocked by overflowing the
!       the store miss buffer. Then the matching stores for all those
!       BIS operations are executed.
!

        sub     %o0, 8, %o0             ! adjust %o0 for ASI alignment
.Lalign_loop:
        cmp     %o5, ST_CHUNK*64
        blu,pt  %xcc, .Lalign_loop_fin
         mov    ST_CHUNK,%o3
.Lalign_loop_start:
        prefetch [%o1 + (ALIGN_PRE * BLOCK_SIZE)], 21
        subcc   %o3, 1, %o3
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5)
        add     %o1, 64, %o1
        add     %o0, 8, %o0
        EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
        bgu     %xcc,.Lalign_loop_start
         add    %o0, 56, %o0

        mov     ST_CHUNK,%o3
        sllx    %o3, 6, %o4             ! ST_CHUNK*64
        sub     %o1, %o4, %o1           ! reset %o1
        sub     %o0, %o4, %o0           ! reset %o0

.Lalign_loop_rest:
        EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5)
        add     %o0, 16, %o0
        EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5)
        add     %o0, 8, %o0
        EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
        subcc   %o3, 1, %o3
        EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5)
        add     %o0, 8, %o0
        EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5)
        add     %o0, 8, %o0
        EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5)
        add     %o0, 8, %o0
        EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5)
        add     %o1, 64, %o1
        add     %o0, 8, %o0
        EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
        add     %o0, 8, %o0
        EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5)
        sub     %o5, 64, %o5
        bgu     %xcc,.Lalign_loop_rest
        ! mark cache line as LRU
         EX_ST(STORE_INIT(%o4, %o0), memcpy_retl_o2_plus_o5_plus_64)

        cmp     %o5, ST_CHUNK*64
        bgu,pt  %xcc, .Lalign_loop_start
         mov    ST_CHUNK,%o3

        cmp     %o5, 0
        beq     .Lalign_done
         nop
.Lalign_loop_fin:
        EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5)
        EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5)
        EX_ST(STORE(stx, %o4, %o0+8+8), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5)
        EX_ST(STORE(stx, %o4, %o0+8+16), memcpy_retl_o2_plus_o5)
        subcc   %o5, 64, %o5
        EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5_64)
        EX_ST(STORE(stx, %o4, %o0+8+24), memcpy_retl_o2_plus_o5_64)
        EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5_64)
        EX_ST(STORE(stx, %o4, %o0+8+32), memcpy_retl_o2_plus_o5_64)
        EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5_64)
        EX_ST(STORE(stx, %o4, %o0+8+40), memcpy_retl_o2_plus_o5_64)
        EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5_64)
        add     %o1, 64, %o1
        EX_ST(STORE(stx, %o4, %o0+8+48), memcpy_retl_o2_plus_o5_64)
        add     %o0, 64, %o0
        EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5_64)
        bgu     %xcc,.Lalign_loop_fin
         EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_o5_64)

.Lalign_done:
        add     %o0, 8, %o0             ! restore %o0 from ASI alignment
        membar  #StoreStore
        sub     %o2, 63, %o2            ! adjust length to allow cc test
        ba      .Lmedl63                ! in .Lmedl63
         nop

        .align 16
        ! Dst is on 8 byte boundary; src is not; remaining count > SMALL_MAX
.Lunalignsetup:
.Lunalignrejoin:
        mov     %g1, %o3        ! save %g1 as VISEntryHalf clobbers it
#ifdef NON_USER_COPY
        VISEntryHalfFast(.Lmedium_vis_entry_fail_cp)
#else
        VISEntryHalf
#endif
        mov     %o3, %g1        ! restore %g1

        set MED_UMAX, %o3
        cmp %o2, %o3            ! check for.Lmedium unaligned limit
        bge,pt  %xcc,.Lunalign_large
         prefetch [%o1 + (4 * BLOCK_SIZE)], 20
        andn    %o2, 0x3f, %o5          ! %o5 is multiple of block size
        and     %o2, 0x3f, %o2          ! residue bytes in %o2
        cmp     %o2, 8                  ! Insure we do not load beyond
        bgt     .Lunalign_adjust        ! end of source buffer
         andn   %o1, 0x7, %o4           ! %o4 has long word aligned src address
        add     %o2, 64, %o2            ! adjust to leave loop
        sub     %o5, 64, %o5            ! early if necessary
.Lunalign_adjust:
        alignaddr %o1, %g0, %g0         ! generate %gsr
        add     %o1, %o5, %o1           ! advance %o1 to after blocks
        EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5)
.Lunalign_loop:
        EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5)
        faligndata %f0, %f2, %f16
        EX_LD_FP(LOAD(ldd, %o4+16, %f4), memcpy_retl_o2_plus_o5)
        subcc   %o5, BLOCK_SIZE, %o5
        EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_64)
        faligndata %f2, %f4, %f18
        EX_LD_FP(LOAD(ldd, %o4+24, %f6), memcpy_retl_o2_plus_o5_plus_56)
        EX_ST_FP(STORE(std, %f18, %o0+8), memcpy_retl_o2_plus_o5_plus_56)
        faligndata %f4, %f6, %f20
        EX_LD_FP(LOAD(ldd, %o4+32, %f8), memcpy_retl_o2_plus_o5_plus_48)
        EX_ST_FP(STORE(std, %f20, %o0+16), memcpy_retl_o2_plus_o5_plus_48)
        faligndata %f6, %f8, %f22
        EX_LD_FP(LOAD(ldd, %o4+40, %f10), memcpy_retl_o2_plus_o5_plus_40)
        EX_ST_FP(STORE(std, %f22, %o0+24), memcpy_retl_o2_plus_o5_plus_40)
        faligndata %f8, %f10, %f24
        EX_LD_FP(LOAD(ldd, %o4+48, %f12), memcpy_retl_o2_plus_o5_plus_32)
        EX_ST_FP(STORE(std, %f24, %o0+32), memcpy_retl_o2_plus_o5_plus_32)
        faligndata %f10, %f12, %f26
        EX_LD_FP(LOAD(ldd, %o4+56, %f14), memcpy_retl_o2_plus_o5_plus_24)
        add     %o4, BLOCK_SIZE, %o4
        EX_ST_FP(STORE(std, %f26, %o0+40), memcpy_retl_o2_plus_o5_plus_24)
        faligndata %f12, %f14, %f28
        EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5_plus_16)
        EX_ST_FP(STORE(std, %f28, %o0+48), memcpy_retl_o2_plus_o5_plus_16)
        faligndata %f14, %f0, %f30
        EX_ST_FP(STORE(std, %f30, %o0+56), memcpy_retl_o2_plus_o5_plus_8)
        add     %o0, BLOCK_SIZE, %o0
        bgu,pt  %xcc, .Lunalign_loop
         prefetch [%o4 + (5 * BLOCK_SIZE)], 20
        ba      .Lunalign_done
         nop

.Lunalign_large:
        andcc   %o0, 0x3f, %o3          ! is dst 64-byte block aligned?
        bz      %xcc, .Lunalignsrc
         sub    %o3, 64, %o3            ! %o3 will be multiple of 8
        neg     %o3                     ! bytes until dest is 64 byte aligned
        sub     %o2, %o3, %o2           ! update cnt with bytes to be moved
        ! Move bytes according to source alignment
        andcc   %o1, 0x1, %o5
        bnz     %xcc, .Lunalignbyte     ! check for byte alignment
         nop
        andcc   %o1, 2, %o5             ! check for half word alignment
        bnz     %xcc, .Lunalignhalf
         nop
        ! Src is word aligned
.Lunalignword:
        EX_LD_FP(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_o3)    ! load 4 bytes
        add     %o1, 8, %o1             ! increase src ptr by 8
        EX_ST_FP(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_o3)  ! and store 4
        subcc   %o3, 8, %o3             ! decrease count by 8
        EX_LD_FP(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_o3_plus_4)! load 4
        add     %o0, 8, %o0             ! increase dst ptr by 8
        bnz     %xcc, .Lunalignword
         EX_ST_FP(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_o3_plus_4)
        ba      .Lunalignsrc
         nop

        ! Src is half-word aligned
.Lunalignhalf:
        EX_LD_FP(LOAD(lduh, %o1, %o4), memcpy_retl_o2_plus_o3)  ! load 2 bytes
        sllx    %o4, 32, %o5            ! shift left
        EX_LD_FP(LOAD(lduw, %o1+2, %o4), memcpy_retl_o2_plus_o3)
        or      %o4, %o5, %o5
        sllx    %o5, 16, %o5
        EX_LD_FP(LOAD(lduh, %o1+6, %o4), memcpy_retl_o2_plus_o3)
        or      %o4, %o5, %o5
        EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3)
        add     %o1, 8, %o1
        subcc   %o3, 8, %o3
        bnz     %xcc, .Lunalignhalf
         add    %o0, 8, %o0
        ba      .Lunalignsrc
         nop

        ! Src is Byte aligned
.Lunalignbyte:
        sub     %o0, %o1, %o0           ! share pointer advance
.Lunalignbyte_loop:
        EX_LD_FP(LOAD(ldub, %o1, %o4), memcpy_retl_o2_plus_o3)
        sllx    %o4, 56, %o5
        EX_LD_FP(LOAD(lduh, %o1+1, %o4), memcpy_retl_o2_plus_o3)
        sllx    %o4, 40, %o4
        or      %o4, %o5, %o5
        EX_LD_FP(LOAD(lduh, %o1+3, %o4), memcpy_retl_o2_plus_o3)
        sllx    %o4, 24, %o4
        or      %o4, %o5, %o5
        EX_LD_FP(LOAD(lduh, %o1+5, %o4), memcpy_retl_o2_plus_o3)
        sllx    %o4,  8, %o4
        or      %o4, %o5, %o5
        EX_LD_FP(LOAD(ldub, %o1+7, %o4), memcpy_retl_o2_plus_o3)
        or      %o4, %o5, %o5
        add     %o0, %o1, %o0
        EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3)
        sub     %o0, %o1, %o0
        subcc   %o3, 8, %o3
        bnz     %xcc, .Lunalignbyte_loop
         add    %o1, 8, %o1
        add     %o0,%o1, %o0            ! restore pointer

        ! Destination is now block (64 byte aligned)
.Lunalignsrc:
        andn    %o2, 0x3f, %o5          ! %o5 is multiple of block size
        and     %o2, 0x3f, %o2          ! residue bytes in %o2
        add     %o2, 64, %o2            ! Insure we do not load beyond
        sub     %o5, 64, %o5            ! end of source buffer

        andn    %o1, 0x7, %o4           ! %o4 has long word aligned src address
        alignaddr %o1, %g0, %g0         ! generate %gsr
        add     %o1, %o5, %o1           ! advance %o1 to after blocks

        EX_LD_FP(LOAD(ldd, %o4, %f14), memcpy_retl_o2_plus_o5)
        add     %o4, 8, %o4
.Lunalign_sloop:
        EX_LD_FP(LOAD(ldd, %o4, %f16), memcpy_retl_o2_plus_o5)
        faligndata %f14, %f16, %f0
        EX_LD_FP(LOAD(ldd, %o4+8, %f18), memcpy_retl_o2_plus_o5)
        faligndata %f16, %f18, %f2
        EX_LD_FP(LOAD(ldd, %o4+16, %f20), memcpy_retl_o2_plus_o5)
        faligndata %f18, %f20, %f4
        EX_ST_FP(STORE(std, %f0, %o0), memcpy_retl_o2_plus_o5)
        subcc   %o5, 64, %o5
        EX_LD_FP(LOAD(ldd, %o4+24, %f22), memcpy_retl_o2_plus_o5_plus_56)
        faligndata %f20, %f22, %f6
        EX_ST_FP(STORE(std, %f2, %o0+8), memcpy_retl_o2_plus_o5_plus_56)
        EX_LD_FP(LOAD(ldd, %o4+32, %f24), memcpy_retl_o2_plus_o5_plus_48)
        faligndata %f22, %f24, %f8
        EX_ST_FP(STORE(std, %f4, %o0+16), memcpy_retl_o2_plus_o5_plus_48)
        EX_LD_FP(LOAD(ldd, %o4+40, %f26), memcpy_retl_o2_plus_o5_plus_40)
        faligndata %f24, %f26, %f10
        EX_ST_FP(STORE(std, %f6, %o0+24), memcpy_retl_o2_plus_o5_plus_40)
        EX_LD_FP(LOAD(ldd, %o4+48, %f28), memcpy_retl_o2_plus_o5_plus_32)
        faligndata %f26, %f28, %f12
        EX_ST_FP(STORE(std, %f8, %o0+32), memcpy_retl_o2_plus_o5_plus_32)
        add     %o4, 64, %o4
        EX_LD_FP(LOAD(ldd, %o4-8, %f30), memcpy_retl_o2_plus_o5_plus_24)
        faligndata %f28, %f30, %f14
        EX_ST_FP(STORE(std, %f10, %o0+40), memcpy_retl_o2_plus_o5_plus_24)
        EX_ST_FP(STORE(std, %f12, %o0+48), memcpy_retl_o2_plus_o5_plus_16)
        add     %o0, 64, %o0
        EX_ST_FP(STORE(std, %f14, %o0-8), memcpy_retl_o2_plus_o5_plus_8)
        fsrc2   %f30, %f14
        bgu,pt  %xcc, .Lunalign_sloop
         prefetch [%o4 + (8 * BLOCK_SIZE)], 20

.Lunalign_done:
        ! Handle trailing bytes, 64 to 127
        ! Dest long word aligned, Src not long word aligned
        cmp     %o2, 15
        bleu    %xcc, .Lunalign_short

         andn   %o2, 0x7, %o5           ! %o5 is multiple of 8
        and     %o2, 0x7, %o2           ! residue bytes in %o2
        add     %o2, 8, %o2
        sub     %o5, 8, %o5             ! insure we do not load past end of src
        andn    %o1, 0x7, %o4           ! %o4 has long word aligned src address
        add     %o1, %o5, %o1           ! advance %o1 to after multiple of 8
        EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5)! fetch partialword
.Lunalign_by8:
        EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5)
        add     %o4, 8, %o4
        faligndata %f0, %f2, %f16
        subcc   %o5, 8, %o5
        EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_8)
        fsrc2   %f2, %f0
        bgu,pt  %xcc, .Lunalign_by8
         add    %o0, 8, %o0

.Lunalign_short:
#ifdef NON_USER_COPY
        VISExitHalfFast
#else
        VISExitHalf
#endif
        ba      .Lsmallrest
         nop

/*
 * This is a special case of nested memcpy. This can happen when kernel
 * calls unaligned memcpy back to back without saving FP registers. We need
 * traps(context switch) to save/restore FP registers. If the kernel calls
 * memcpy without this trap sequence we will hit FP corruption. Let's use
 * the normal integer load/store method in this case.
 */

#ifdef NON_USER_COPY
.Lmedium_vis_entry_fail_cp:
        or      %o0, %o1, %g2
#endif
.Lmedium_cp:
        LOAD(prefetch, %o1 + 0x40, #n_reads_strong)
        andcc   %g2, 0x7, %g0
        bne,pn  %xcc, .Lmedium_unaligned_cp
         nop

.Lmedium_noprefetch_cp:
        andncc  %o2, 0x20 - 1, %o5
        be,pn   %xcc, 2f
         sub    %o2, %o5, %o2
1:      EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1 + 0x10, %g7), memcpy_retl_o2_plus_o5)
        EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5)
        add     %o1, 0x20, %o1
        subcc   %o5, 0x20, %o5
        EX_ST(STORE(stx, %o3, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32)
        EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24)
        EX_ST(STORE(stx, %g7, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_16)
        EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8)
        bne,pt  %xcc, 1b
         add    %o0, 0x20, %o0
2:      andcc   %o2, 0x18, %o5
        be,pt   %xcc, 3f
         sub    %o2, %o5, %o2
1:      EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5)
        add     %o1, 0x08, %o1
        add     %o0, 0x08, %o0
        subcc   %o5, 0x08, %o5
        bne,pt  %xcc, 1b
         EX_ST(STORE(stx, %o3, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8)
3:      brz,pt  %o2, .Lexit_cp
         cmp    %o2, 0x04
        bl,pn   %xcc, .Ltiny_cp
         nop
        EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2)
        add     %o1, 0x04, %o1
        add     %o0, 0x04, %o0
        subcc   %o2, 0x04, %o2
        bne,pn  %xcc, .Ltiny_cp
         EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_4)
        ba,a,pt %xcc, .Lexit_cp

.Lmedium_unaligned_cp:
        /* First get dest 8 byte aligned.  */
        sub     %g0, %o0, %o3
        and     %o3, 0x7, %o3
        brz,pt  %o3, 2f
         sub    %o2, %o3, %o2

1:      EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_o3)
        add     %o1, 1, %o1
        subcc   %o3, 1, %o3
        add     %o0, 1, %o0
        bne,pt  %xcc, 1b
         EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_o3_plus_1)
2:
        and     %o1, 0x7, %o3
        brz,pn  %o3, .Lmedium_noprefetch_cp
         sll    %o3, 3, %o3
        mov     64, %g2
        sub     %g2, %o3, %g2
        andn    %o1, 0x7, %o1
        EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2)
        sllx    %o4, %o3, %o4
        andn    %o2, 0x08 - 1, %o5
        sub     %o2, %o5, %o2

1:      EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5)
        add     %o1, 0x08, %o1
        subcc   %o5, 0x08, %o5
        srlx    %g3, %g2, %g7
        or      %g7, %o4, %g7
        EX_ST(STORE(stx, %g7, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8)
        add     %o0, 0x08, %o0
        bne,pt  %xcc, 1b
         sllx   %g3, %o3, %o4
        srl     %o3, 3, %o3
        add     %o1, %o3, %o1
        brz,pn  %o2, .Lexit_cp
         nop
        ba,pt   %xcc, .Lsmall_unaligned_cp

.Ltiny_cp:
        EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2)
        subcc   %o2, 1, %o2
        be,pn   %xcc, .Lexit_cp
         EX_ST(STORE(stb, %o3, %o0 + 0x00), memcpy_retl_o2_plus_1)
        EX_LD(LOAD(ldub, %o1 + 0x01, %o3), memcpy_retl_o2)
        subcc   %o2, 1, %o2
        be,pn   %xcc, .Lexit_cp
         EX_ST(STORE(stb, %o3, %o0 + 0x01), memcpy_retl_o2_plus_1)
        EX_LD(LOAD(ldub, %o1 + 0x02, %o3), memcpy_retl_o2)
        ba,pt   %xcc, .Lexit_cp
         EX_ST(STORE(stb, %o3, %o0 + 0x02), memcpy_retl_o2)

.Lsmall_cp:
        andcc   %g2, 0x3, %g0
        bne,pn  %xcc, .Lsmall_unaligned_cp
         andn   %o2, 0x4 - 1, %o5
        sub     %o2, %o5, %o2
1:
        EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5)
        add     %o1, 0x04, %o1
        subcc   %o5, 0x04, %o5
        add     %o0, 0x04, %o0
        bne,pt  %xcc, 1b
         EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4)
        brz,pt  %o2, .Lexit_cp
         nop
        ba,a,pt %xcc, .Ltiny_cp

.Lsmall_unaligned_cp:
1:      EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2)
        add     %o1, 1, %o1
        add     %o0, 1, %o0
        subcc   %o2, 1, %o2
        bne,pt  %xcc, 1b
         EX_ST(STORE(stb, %o3, %o0 - 0x01), memcpy_retl_o2_plus_1)
        ba,a,pt %xcc, .Lexit_cp

.Lsmallrest:
        tst     %o2
        bz,pt   %xcc, .Lsmallx
         cmp    %o2, 4
        blt,pn  %xcc, .Lsmallleft3
         nop
        sub     %o2, 3, %o2
.Lsmallnotalign4:
        EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_3)! read byte
        subcc   %o2, 4, %o2             ! reduce count by 4
        EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_7)! write byte & repeat
        EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2_plus_6)! for total of 4
        add     %o1, 4, %o1             ! advance SRC by 4
        EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_6)
        EX_LD(LOAD(ldub, %o1-2, %o3), memcpy_retl_o2_plus_5)
        add     %o0, 4, %o0             ! advance DST by 4
        EX_ST(STORE(stb, %o3, %o0-2), memcpy_retl_o2_plus_5)
        EX_LD(LOAD(ldub, %o1-1, %o3), memcpy_retl_o2_plus_4)
        bgu,pt  %xcc, .Lsmallnotalign4  ! loop til 3 or fewer bytes remain
        EX_ST(STORE(stb, %o3, %o0-1), memcpy_retl_o2_plus_4)
        addcc   %o2, 3, %o2             ! restore count
        bz,pt   %xcc, .Lsmallx
.Lsmallleft3:                           ! 1, 2, or 3 bytes remain
        subcc   %o2, 1, %o2
        EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_1)      ! load one byte
        bz,pt   %xcc, .Lsmallx
        EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_1)      ! store one byte
        EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2)   ! load second byte
        subcc   %o2, 1, %o2
        bz,pt   %xcc, .Lsmallx
        EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_1)! store second byte
        EX_LD(LOAD(ldub, %o1+2, %o3), memcpy_retl_o2)   ! load third byte
        EX_ST(STORE(stb, %o3, %o0+2), memcpy_retl_o2)   ! store third byte
.Lsmallx:
        retl
         mov    EX_RETVAL(%g1), %o0
.Lsmallfin:
        tst     %o2
        bnz,pn  %xcc, .Lsmallleft3
         nop
        retl
         mov    EX_RETVAL(%g1), %o0     ! restore %o0
.Lexit_cp:
        retl
         mov    EX_RETVAL(%g1), %o0
        .size  FUNC_NAME, .-FUNC_NAME