/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

        .file   "atomic.s"

#include <sys/asm_linkage.h>

#if defined(_KERNEL)
        /*
         * Legacy kernel interfaces; they will go away the moment our closed
         * bins no longer require them.
         */
        ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
        ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
        ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
        ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
        ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
        ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
        ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
        ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function)
#endif

        /*
         * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_inc_8_nv.
         */
        ENTRY(atomic_inc_8)
        ALTENTRY(atomic_inc_8_nv)
        ALTENTRY(atomic_inc_uchar)
        ALTENTRY(atomic_inc_uchar_nv)
        ba      add_8
          add   %g0, 1, %o1
        SET_SIZE(atomic_inc_uchar_nv)
        SET_SIZE(atomic_inc_uchar)
        SET_SIZE(atomic_inc_8_nv)
        SET_SIZE(atomic_inc_8)

        /*
         * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_dec_8_nv.
         */
        ENTRY(atomic_dec_8)
        ALTENTRY(atomic_dec_8_nv)
        ALTENTRY(atomic_dec_uchar)
        ALTENTRY(atomic_dec_uchar_nv)
        ba      add_8
          sub   %g0, 1, %o1
        SET_SIZE(atomic_dec_uchar_nv)
        SET_SIZE(atomic_dec_uchar)
        SET_SIZE(atomic_dec_8_nv)
        SET_SIZE(atomic_dec_8)

        /*
         * NOTE: If atomic_add_8 and atomic_add_8_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_add_8_nv.
         */
        ENTRY(atomic_add_8)
        ALTENTRY(atomic_add_8_nv)
        ALTENTRY(atomic_add_char)
        ALTENTRY(atomic_add_char_nv)
add_8:
        and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        set     0xff, %o3               ! %o3 = mask
        sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single byte value
        andn    %o0, 0x3, %o0           ! %o0 = word address
        ld      [%o0], %o2              ! read old value
1:
        add     %o2, %o1, %o5           ! add value to the old value
        and     %o5, %o3, %o5           ! clear other bits
        andn    %o2, %o3, %o4           ! clear target bits
        or      %o4, %o5, %o5           ! insert the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        add     %o2, %o1, %o5
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = new value
        SET_SIZE(atomic_add_char_nv)
        SET_SIZE(atomic_add_char)
        SET_SIZE(atomic_add_8_nv)
        SET_SIZE(atomic_add_8)

        /*
         * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_inc_16_nv.
         */
        ENTRY(atomic_inc_16)
        ALTENTRY(atomic_inc_16_nv)
        ALTENTRY(atomic_inc_ushort)
        ALTENTRY(atomic_inc_ushort_nv)
        ba      add_16
          add   %g0, 1, %o1
        SET_SIZE(atomic_inc_ushort_nv)
        SET_SIZE(atomic_inc_ushort)
        SET_SIZE(atomic_inc_16_nv)
        SET_SIZE(atomic_inc_16)

        /*
         * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_dec_16_nv.
         */
        ENTRY(atomic_dec_16)
        ALTENTRY(atomic_dec_16_nv)
        ALTENTRY(atomic_dec_ushort)
        ALTENTRY(atomic_dec_ushort_nv)
        ba      add_16
          sub   %g0, 1, %o1
        SET_SIZE(atomic_dec_ushort_nv)
        SET_SIZE(atomic_dec_ushort)
        SET_SIZE(atomic_dec_16_nv)
        SET_SIZE(atomic_dec_16)

        /*
         * NOTE: If atomic_add_16 and atomic_add_16_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_add_16_nv.
         */
        ENTRY(atomic_add_16)
        ALTENTRY(atomic_add_16_nv)
        ALTENTRY(atomic_add_short)
        ALTENTRY(atomic_add_short_nv)
add_16:
        and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
        sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        sethi   %hi(0xffff0000), %o3    ! %o3 = mask
        srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single short value
        andn    %o0, 0x2, %o0           ! %o0 = word address
        ! if low-order bit is 1, we will properly get an alignment fault here
        ld      [%o0], %o2              ! read old value
1:
        add     %o1, %o2, %o5           ! add value to the old value
        and     %o5, %o3, %o5           ! clear other bits
        andn    %o2, %o3, %o4           ! clear target bits
        or      %o4, %o5, %o5           ! insert the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        add     %o1, %o2, %o5
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = new value
        SET_SIZE(atomic_add_short_nv)
        SET_SIZE(atomic_add_short)
        SET_SIZE(atomic_add_16_nv)
        SET_SIZE(atomic_add_16)

        /*
         * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_inc_32_nv.
         */
        ENTRY(atomic_inc_32)
        ALTENTRY(atomic_inc_32_nv)
        ALTENTRY(atomic_inc_uint)
        ALTENTRY(atomic_inc_uint_nv)
        ALTENTRY(atomic_inc_ulong)
        ALTENTRY(atomic_inc_ulong_nv)
        ba      add_32
          add   %g0, 1, %o1
        SET_SIZE(atomic_inc_ulong_nv)
        SET_SIZE(atomic_inc_ulong)
        SET_SIZE(atomic_inc_uint_nv)
        SET_SIZE(atomic_inc_uint)
        SET_SIZE(atomic_inc_32_nv)
        SET_SIZE(atomic_inc_32)

        /*
         * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_dec_32_nv.
         */
        ENTRY(atomic_dec_32)
        ALTENTRY(atomic_dec_32_nv)
        ALTENTRY(atomic_dec_uint)
        ALTENTRY(atomic_dec_uint_nv)
        ALTENTRY(atomic_dec_ulong)
        ALTENTRY(atomic_dec_ulong_nv)
        ba      add_32
          sub   %g0, 1, %o1
        SET_SIZE(atomic_dec_ulong_nv)
        SET_SIZE(atomic_dec_ulong)
        SET_SIZE(atomic_dec_uint_nv)
        SET_SIZE(atomic_dec_uint)
        SET_SIZE(atomic_dec_32_nv)
        SET_SIZE(atomic_dec_32)

        /*
         * NOTE: If atomic_add_32 and atomic_add_32_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_add_32_nv.
         */
        ENTRY(atomic_add_32)
        ALTENTRY(atomic_add_32_nv)
        ALTENTRY(atomic_add_int)
        ALTENTRY(atomic_add_int_nv)
        ALTENTRY(atomic_add_ptr)
        ALTENTRY(atomic_add_ptr_nv)
        ALTENTRY(atomic_add_long)
        ALTENTRY(atomic_add_long_nv)
add_32:
        ld      [%o0], %o2
1:
        add     %o2, %o1, %o3
        cas     [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %icc, 1b
          mov   %o3, %o2
        retl
        add     %o2, %o1, %o0           ! return new value
        SET_SIZE(atomic_add_long_nv)
        SET_SIZE(atomic_add_long)
        SET_SIZE(atomic_add_ptr_nv)
        SET_SIZE(atomic_add_ptr)
        SET_SIZE(atomic_add_int_nv)
        SET_SIZE(atomic_add_int)
        SET_SIZE(atomic_add_32_nv)
        SET_SIZE(atomic_add_32)

        /*
         * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_inc_64_nv.
         */
        ENTRY(atomic_inc_64)
        ALTENTRY(atomic_inc_64_nv)
        ba      add_64
          add   %g0, 1, %o1
        SET_SIZE(atomic_inc_64_nv)
        SET_SIZE(atomic_inc_64)

        /*
         * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_dec_64_nv.
         */
        ENTRY(atomic_dec_64)
        ALTENTRY(atomic_dec_64_nv)
        ba      add_64
          sub   %g0, 1, %o1
        SET_SIZE(atomic_dec_64_nv)
        SET_SIZE(atomic_dec_64)

        /*
         * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_add_64_nv.
         */
        ENTRY(atomic_add_64)
        ALTENTRY(atomic_add_64_nv)
        sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
        srl     %o2, 0, %o2
        add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
add_64:
        ldx     [%o0], %o2
1:
        add     %o2, %o1, %o3
        casx    [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %xcc, 1b
          mov   %o3, %o2
        add     %o2, %o1, %o1           ! return lower 32-bits in %o1
        retl
        srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
        SET_SIZE(atomic_add_64_nv)
        SET_SIZE(atomic_add_64)

        /*
         * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_or_8_nv.
         */
        ENTRY(atomic_or_8)
        ALTENTRY(atomic_or_8_nv)
        ALTENTRY(atomic_or_uchar)
        ALTENTRY(atomic_or_uchar_nv)
        and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        set     0xff, %o3               ! %o3 = mask
        sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single byte value
        andn    %o0, 0x3, %o0           ! %o0 = word address
        ld      [%o0], %o2              ! read old value
1:
        or      %o2, %o1, %o5           ! or in the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        or      %o2, %o1, %o5
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = new value
        SET_SIZE(atomic_or_uchar_nv)
        SET_SIZE(atomic_or_uchar)
        SET_SIZE(atomic_or_8_nv)
        SET_SIZE(atomic_or_8)

        /*
         * NOTE: If atomic_or_16 and atomic_or_16_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_or_16_nv.
         */
        ENTRY(atomic_or_16)
        ALTENTRY(atomic_or_16_nv)
        ALTENTRY(atomic_or_ushort)
        ALTENTRY(atomic_or_ushort_nv)
        and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
        sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        sethi   %hi(0xffff0000), %o3    ! %o3 = mask
        srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single short value
        andn    %o0, 0x2, %o0           ! %o0 = word address
        ! if low-order bit is 1, we will properly get an alignment fault here
        ld      [%o0], %o2              ! read old value
1:
        or      %o2, %o1, %o5           ! or in the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        or      %o2, %o1, %o5           ! or in the new value
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = new value
        SET_SIZE(atomic_or_ushort_nv)
        SET_SIZE(atomic_or_ushort)
        SET_SIZE(atomic_or_16_nv)
        SET_SIZE(atomic_or_16)

        /*
         * NOTE: If atomic_or_32 and atomic_or_32_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_or_32_nv.
         */
        ENTRY(atomic_or_32)
        ALTENTRY(atomic_or_32_nv)
        ALTENTRY(atomic_or_uint)
        ALTENTRY(atomic_or_uint_nv)
        ALTENTRY(atomic_or_ulong)
        ALTENTRY(atomic_or_ulong_nv)
        ld      [%o0], %o2
1:
        or      %o2, %o1, %o3
        cas     [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %icc, 1b
          mov   %o3, %o2
        retl
        or      %o2, %o1, %o0           ! return new value
        SET_SIZE(atomic_or_ulong_nv)
        SET_SIZE(atomic_or_ulong)
        SET_SIZE(atomic_or_uint_nv)
        SET_SIZE(atomic_or_uint)
        SET_SIZE(atomic_or_32_nv)
        SET_SIZE(atomic_or_32)

        /*
         * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_or_64_nv.
         */
        ENTRY(atomic_or_64)
        ALTENTRY(atomic_or_64_nv)
        sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
        srl     %o2, 0, %o2
        add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
        ldx     [%o0], %o2
1:
        or      %o2, %o1, %o3
        casx    [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %xcc, 1b
          mov   %o3, %o2
        or      %o2, %o1, %o1           ! return lower 32-bits in %o1
        retl
        srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
        SET_SIZE(atomic_or_64_nv)
        SET_SIZE(atomic_or_64)

        /*
         * NOTE: If atomic_and_8 and atomic_and_8_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_and_8_nv.
         */
        ENTRY(atomic_and_8)
        ALTENTRY(atomic_and_8_nv)
        ALTENTRY(atomic_and_uchar)
        ALTENTRY(atomic_and_uchar_nv)
        and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        set     0xff, %o3               ! %o3 = mask
        sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        orn     %o1, %o3, %o1           ! all ones in other bytes
        andn    %o0, 0x3, %o0           ! %o0 = word address
        ld      [%o0], %o2              ! read old value
1:
        and     %o2, %o1, %o5           ! and in the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        and     %o2, %o1, %o5
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = new value
        SET_SIZE(atomic_and_uchar_nv)
        SET_SIZE(atomic_and_uchar)
        SET_SIZE(atomic_and_8_nv)
        SET_SIZE(atomic_and_8)

        /*
         * NOTE: If atomic_and_16 and atomic_and_16_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_and_16_nv.
         */
        ENTRY(atomic_and_16)
        ALTENTRY(atomic_and_16_nv)
        ALTENTRY(atomic_and_ushort)
        ALTENTRY(atomic_and_ushort_nv)
        and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
        sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        sethi   %hi(0xffff0000), %o3    ! %o3 = mask
        srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        orn     %o1, %o3, %o1           ! all ones in the other half
        andn    %o0, 0x2, %o0           ! %o0 = word address
        ! if low-order bit is 1, we will properly get an alignment fault here
        ld      [%o0], %o2              ! read old value
1:
        and     %o2, %o1, %o5           ! and in the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        and     %o2, %o1, %o5
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = new value
        SET_SIZE(atomic_and_ushort_nv)
        SET_SIZE(atomic_and_ushort)
        SET_SIZE(atomic_and_16_nv)
        SET_SIZE(atomic_and_16)

        /*
         * NOTE: If atomic_and_32 and atomic_and_32_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_and_32_nv.
         */
        ENTRY(atomic_and_32)
        ALTENTRY(atomic_and_32_nv)
        ALTENTRY(atomic_and_uint)
        ALTENTRY(atomic_and_uint_nv)
        ALTENTRY(atomic_and_ulong)
        ALTENTRY(atomic_and_ulong_nv)
        ld      [%o0], %o2
1:
        and     %o2, %o1, %o3
        cas     [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %icc, 1b
          mov   %o3, %o2
        retl
        and     %o2, %o1, %o0           ! return new value
        SET_SIZE(atomic_and_ulong_nv)
        SET_SIZE(atomic_and_ulong)
        SET_SIZE(atomic_and_uint_nv)
        SET_SIZE(atomic_and_uint)
        SET_SIZE(atomic_and_32_nv)
        SET_SIZE(atomic_and_32)

        /*
         * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
         * separated, you need to also edit the libc sparc platform
         * specific mapfile and remove the NODYNSORT attribute
         * from atomic_and_64_nv.
         */
        ENTRY(atomic_and_64)
        ALTENTRY(atomic_and_64_nv)
        sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
        srl     %o2, 0, %o2
        add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
        ldx     [%o0], %o2
1:
        and     %o2, %o1, %o3
        casx    [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %xcc, 1b
          mov   %o3, %o2
        and     %o2, %o1, %o1           ! return lower 32-bits in %o1
        retl
        srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
        SET_SIZE(atomic_and_64_nv)
        SET_SIZE(atomic_and_64)

        ENTRY(atomic_cas_8)
        ALTENTRY(atomic_cas_uchar)
        and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        set     0xff, %o3               ! %o3 = mask
        sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single byte value
        sll     %o2, %g1, %o2           ! %o2 = shifted to bit offset
        and     %o2, %o3, %o2           ! %o2 = single byte value
        andn    %o0, 0x3, %o0           ! %o0 = word address
        ld      [%o0], %o4              ! read old value
1:
        andn    %o4, %o3, %o4           ! clear target bits
        or      %o4, %o2, %o5           ! insert the new value
        or      %o4, %o1, %o4           ! insert the comparison value
        cas     [%o0], %o4, %o5
        cmp     %o4, %o5                ! did we succeed?
        be,pt   %icc, 2f
          and   %o5, %o3, %o4           ! isolate the old value
        cmp     %o1, %o4                ! should we have succeeded?
        be,a,pt %icc, 1b                ! yes, try again
          mov   %o5, %o4                ! %o4 = old value
2:
        retl
        srl     %o4, %g1, %o0           ! %o0 = old value
        SET_SIZE(atomic_cas_uchar)
        SET_SIZE(atomic_cas_8)

        ENTRY(atomic_cas_16)
        ALTENTRY(atomic_cas_ushort)
        and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
        sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        sethi   %hi(0xffff0000), %o3    ! %o3 = mask
        srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single short value
        sll     %o2, %g1, %o2           ! %o2 = shifted to bit offset
        and     %o2, %o3, %o2           ! %o2 = single short value
        andn    %o0, 0x2, %o0           ! %o0 = word address
        ! if low-order bit is 1, we will properly get an alignment fault here
        ld      [%o0], %o4              ! read old value
1:
        andn    %o4, %o3, %o4           ! clear target bits
        or      %o4, %o2, %o5           ! insert the new value
        or      %o4, %o1, %o4           ! insert the comparison value
        cas     [%o0], %o4, %o5
        cmp     %o4, %o5                ! did we succeed?
        be,pt   %icc, 2f
          and   %o5, %o3, %o4           ! isolate the old value
        cmp     %o1, %o4                ! should we have succeeded?
        be,a,pt %icc, 1b                ! yes, try again
          mov   %o5, %o4                ! %o4 = old value
2:
        retl
        srl     %o4, %g1, %o0           ! %o0 = old value
        SET_SIZE(atomic_cas_ushort)
        SET_SIZE(atomic_cas_16)

        ENTRY(atomic_cas_32)
        ALTENTRY(atomic_cas_uint)
        ALTENTRY(atomic_cas_ptr)
        ALTENTRY(atomic_cas_ulong)
        cas     [%o0], %o1, %o2
        retl
        mov     %o2, %o0
        SET_SIZE(atomic_cas_ulong)
        SET_SIZE(atomic_cas_ptr)
        SET_SIZE(atomic_cas_uint)
        SET_SIZE(atomic_cas_32)

        ENTRY(atomic_cas_64)
        sllx    %o1, 32, %o1            ! cmp's upper 32 in %o1, lower in %o2
        srl     %o2, 0, %o2             ! convert 2 32-bit args into 1 64-bit
        add     %o1, %o2, %o1
        sllx    %o3, 32, %o2            ! newval upper 32 in %o3, lower in %o4
        srl     %o4, 0, %o4             ! setup %o2 to have newval
        add     %o2, %o4, %o2
        casx    [%o0], %o1, %o2
        srl     %o2, 0, %o1             ! return lower 32-bits in %o1
        retl
        srlx    %o2, 32, %o0            ! return upper 32-bits in %o0
        SET_SIZE(atomic_cas_64)

        ENTRY(atomic_swap_8)
        ALTENTRY(atomic_swap_uchar)
        and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        set     0xff, %o3               ! %o3 = mask
        sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single byte value
        andn    %o0, 0x3, %o0           ! %o0 = word address
        ld      [%o0], %o2              ! read old value
1:
        andn    %o2, %o3, %o5           ! clear target bits
        or      %o5, %o1, %o5           ! insert the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = old value
        SET_SIZE(atomic_swap_uchar)
        SET_SIZE(atomic_swap_8)

        ENTRY(atomic_swap_16)
        ALTENTRY(atomic_swap_ushort)
        and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
        xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
        sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
        sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
        sethi   %hi(0xffff0000), %o3    ! %o3 = mask
        srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
        sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
        and     %o1, %o3, %o1           ! %o1 = single short value
        andn    %o0, 0x2, %o0           ! %o0 = word address
        ! if low-order bit is 1, we will properly get an alignment fault here
        ld      [%o0], %o2              ! read old value
1:
        andn    %o2, %o3, %o5           ! clear target bits
        or      %o5, %o1, %o5           ! insert the new value
        cas     [%o0], %o2, %o5
        cmp     %o2, %o5
        bne,a,pn %icc, 1b
          mov   %o5, %o2                ! %o2 = old value
        and     %o5, %o3, %o5
        retl
        srl     %o5, %g1, %o0           ! %o0 = old value
        SET_SIZE(atomic_swap_ushort)
        SET_SIZE(atomic_swap_16)

        ENTRY(atomic_swap_32)
        ALTENTRY(atomic_swap_uint)
        ALTENTRY(atomic_swap_ptr)
        ALTENTRY(atomic_swap_ulong)
        ld      [%o0], %o2
1:
        mov     %o1, %o3
        cas     [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %icc, 1b
          mov   %o3, %o2
        retl
        mov     %o3, %o0
        SET_SIZE(atomic_swap_ulong)
        SET_SIZE(atomic_swap_ptr)
        SET_SIZE(atomic_swap_uint)
        SET_SIZE(atomic_swap_32)

        ENTRY(atomic_swap_64)
        sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
        srl     %o2, 0, %o2
        add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
        ldx     [%o0], %o2
1:
        mov     %o1, %o3
        casx    [%o0], %o2, %o3
        cmp     %o2, %o3
        bne,a,pn %xcc, 1b
          mov   %o3, %o2
        srl     %o3, 0, %o1             ! return lower 32-bits in %o1
        retl
        srlx    %o3, 32, %o0            ! return upper 32-bits in %o0
        SET_SIZE(atomic_swap_64)

        ENTRY(atomic_set_long_excl)
        mov     1, %o3
        slln    %o3, %o1, %o3
        ldn     [%o0], %o2
1:
        andcc   %o2, %o3, %g0           ! test if the bit is set
        bnz,a,pn %ncc, 2f               ! if so, then fail out
          mov   -1, %o0
        or      %o2, %o3, %o4           ! set the bit, and try to commit it
        casn    [%o0], %o2, %o4
        cmp     %o2, %o4
        bne,a,pn %ncc, 1b               ! failed to commit, try again
          mov   %o4, %o2
        mov     %g0, %o0
2:
        retl
        nop
        SET_SIZE(atomic_set_long_excl)

        ENTRY(atomic_clear_long_excl)
        mov     1, %o3
        slln    %o3, %o1, %o3
        ldn     [%o0], %o2
1:
        andncc  %o3, %o2, %g0           ! test if the bit is clear
        bnz,a,pn %ncc, 2f               ! if so, then fail out
          mov   -1, %o0
        andn    %o2, %o3, %o4           ! clear the bit, and try to commit it
        casn    [%o0], %o2, %o4
        cmp     %o2, %o4
        bne,a,pn %ncc, 1b               ! failed to commit, try again
          mov   %o4, %o2
        mov     %g0, %o0
2:
        retl
        nop
        SET_SIZE(atomic_clear_long_excl)

#if !defined(_KERNEL)

        /*
         * Spitfires and Blackbirds have a problem with membars in the
         * delay slot (SF_ERRATA_51).  For safety's sake, we assume
         * that the whole world needs the workaround.
         */
        ENTRY(membar_enter)
        membar  #StoreLoad|#StoreStore
        retl
        nop
        SET_SIZE(membar_enter)

        ENTRY(membar_exit)
        membar  #LoadStore|#StoreStore
        retl
        nop
        SET_SIZE(membar_exit)

        ENTRY(membar_producer)
        membar  #StoreStore
        retl
        nop
        SET_SIZE(membar_producer)

        ENTRY(membar_consumer)
        membar  #LoadLoad
        retl
        nop
        SET_SIZE(membar_consumer)

#endif  /* !_KERNEL */