/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/asi.h>
#include <sys/asm_linkage.h>
#include <sys/machthread.h>
#include <sys/privregs.h>
#include <sys/ontrap.h>
#include <sys/dditypes.h>

#include "assym.h"

/*
 * This file implements the following ddi common access
 * functions:
 *
 *      ddi_get{8,16,32,64}
 *      ddi_put{8,16,32,64}
 *
 * and the underlying "trivial" implementations
 *
 *      i_ddi_{get,put}{8,16,32,64}
 *
 * which assume that there is no need to check the access handle -
 * byte swapping will be done by the mmu and the address is always
 * accessible via ld/st instructions.
 */

/*
 * The functionality of each of the ddi_get/put routines is performed by
 * the respective indirect function defined in the access handle.  Use of
 * the access handle functions provides compatibility across platforms for
 * drivers.
 *
 * By default, the indirect access handle functions are initialized to the
 * i_ddi_get/put routines to perform memory mapped IO.  If memory mapped IO
 * is not possible or desired, the access handle must be intialized to another
 * valid routine to perform the sepcified IO operation.
 *
 * The alignment and placement of the following functions have been optimized
 * such that the implementation specific versions, i_ddi*, fall within the
 * same cache-line of the generic versions, ddi_*.  This insures that an
 * I-cache hit will occur thus minimizing the performance impact of using the
 * access handle.
 */

        .align 32
        ENTRY(ddi_get8)
        ALTENTRY(ddi_io_get8)
        ALTENTRY(ddi_mem_get8)
        ldn      [%o0 + AHI_GET8], %g1   /* hdl->ahi_get8 access hndl */
        jmpl    %g1, %g0                 /* jump to access handle routine */
        nop
        SET_SIZE(ddi_get8)
        SET_SIZE(ddi_io_get8)
        SET_SIZE(ddi_mem_get8)

        .align 16
        ENTRY(i_ddi_get8)
        retl
        ldub    [%o1], %o0
        SET_SIZE(i_ddi_get8)

        .align 32
        ENTRY(ddi_get16)
        ALTENTRY(ddi_io_get16)
        ALTENTRY(ddi_mem_get16)
        ldn      [%o0 + AHI_GET16], %g1   /* hdl->ahi_get16 access hndl */
        jmpl    %g1, %g0                  /* jump to access handle routine */
        nop
        SET_SIZE(ddi_get16)
        SET_SIZE(ddi_io_get16)
        SET_SIZE(ddi_mem_get16)

        .align 16
        ENTRY(i_ddi_get16)
        ALTENTRY(i_ddi_swap_get16)
        retl
        lduh    [%o1], %o0
        SET_SIZE(i_ddi_get16)
        SET_SIZE(i_ddi_swap_get16)

        .align 32
        ENTRY(ddi_get32)
        ALTENTRY(ddi_io_get32)
        ALTENTRY(ddi_mem_get32)
        ldn      [%o0 + AHI_GET32], %g1   /* hdl->ahi_get32 access handle */
        jmpl    %g1, %g0                  /* jump to access handle routine */
        nop
        SET_SIZE(ddi_get32)
        SET_SIZE(ddi_io_get32)
        SET_SIZE(ddi_mem_get32)

        .align 16
        ENTRY(i_ddi_get32)
        ALTENTRY(i_ddi_swap_get32)
        retl
        ld      [%o1], %o0
        SET_SIZE(i_ddi_get32)
        SET_SIZE(i_ddi_swap_get32)

        .align 32
        ENTRY(ddi_get64)
        ALTENTRY(ddi_io_get64)
        ALTENTRY(ddi_mem_get64)
        ldn      [%o0 + AHI_GET64], %g1   /* hdl->ahi_get64 access handle */
        jmpl    %g1, %g0                  /* jump to access handle routine */
        nop
        SET_SIZE(ddi_get64)
        SET_SIZE(ddi_io_get64)
        SET_SIZE(ddi_mem_get64)

        .align 16
        ENTRY(i_ddi_get64)
        ALTENTRY(i_ddi_swap_get64)
        retl
        ldx     [%o1], %o0
        SET_SIZE(i_ddi_get64)
        SET_SIZE(i_ddi_swap_get64)

        .align 32
        ENTRY(ddi_put8)
        ALTENTRY(ddi_io_put8)
        ALTENTRY(ddi_mem_put8)
        ldn      [%o0 + AHI_PUT8], %g1   /* hdl->ahi_put8 access handle */
        jmpl    %g1, %g0                 /* jump to access handle routine */
        nop
        SET_SIZE(ddi_put8)
        SET_SIZE(ddi_io_put8)
        SET_SIZE(ddi_mem_put8)

        .align 16
        ENTRY(i_ddi_put8)
        retl
        stub    %o2, [%o1]
        SET_SIZE(i_ddi_put8)

        .align 32
        ENTRY(ddi_put16)
        ALTENTRY(ddi_io_put16)
        ALTENTRY(ddi_mem_put16)
        ldn      [%o0 + AHI_PUT16], %g1   /* hdl->ahi_put16 access handle */
        jmpl    %g1, %g0                  /* jump to access handle routine */
        nop
        SET_SIZE(ddi_put16)
        SET_SIZE(ddi_io_put16)
        SET_SIZE(ddi_mem_put16)

        .align 16
        ENTRY(i_ddi_put16)
        ALTENTRY(i_ddi_swap_put16)
        retl
        stuh    %o2, [%o1]
        SET_SIZE(i_ddi_put16)
        SET_SIZE(i_ddi_swap_put16)

        .align 32
        ENTRY(ddi_put32)
        ALTENTRY(ddi_io_put32)
        ALTENTRY(ddi_mem_put32)
        ldn      [%o0 + AHI_PUT32], %g1   /* hdl->ahi_put16 access handle */
        jmpl    %g1, %g0                  /* jump to access handle routine */
        nop
        SET_SIZE(ddi_put32)
        SET_SIZE(ddi_io_put32)
        SET_SIZE(ddi_mem_put32)

        .align 16
        ENTRY(i_ddi_put32)
        ALTENTRY(i_ddi_swap_put32)
        retl
        st      %o2, [%o1]
        SET_SIZE(i_ddi_put32)
        SET_SIZE(i_ddi_swap_put32)

        .align 32
        ENTRY(ddi_put64)
        ALTENTRY(ddi_io_put64)
        ALTENTRY(ddi_mem_put64)
        ldn      [%o0 + AHI_PUT64], %g1   /* hdl->ahi_put64 access handle */
        jmpl    %g1, %g0                  /* jump to access handle routine */
        nop
        SET_SIZE(ddi_put64)
        SET_SIZE(ddi_io_put64)
        SET_SIZE(ddi_mem_put64)

        .align 16
        ENTRY(i_ddi_put64)
        ALTENTRY(i_ddi_swap_put64)
        retl
        stx     %o2, [%o1]
        SET_SIZE(i_ddi_put64)
        SET_SIZE(i_ddi_swap_put64)

/*
 * The ddi_io_rep_get/put routines don't take a flag argument like the "plain"
 * and mem versions do.  This flag is used to determine whether or not the
 * device address or port should be automatically incremented.  For IO space,
 * the device port is never incremented and as such, the flag is always set
 * to DDI_DEV_NO_AUTOINCR.
 *
 * This define processes the repetitive get functionality.  Automatic
 * incrementing of the device address is determined by the flag field
 * %o4.  If this is set for AUTOINCR, %o4 is updated with 1 for the
 * subsequent increment in 2:.
 *
 * If this flag is not set for AUTOINCR, %o4 is update with a value of 0 thus
 * making the increment operation a non-operation.
 */

#define DDI_REP_GET(n,s)                        \
        cmp     DDI_DEV_NO_AUTOINCR, %o4;       \
        mov     %g0, %o4;                       \
        brz,pn  %o3, 1f;                        \
        movnz   %xcc, n, %o4;                   \
2:                                              \
        dec     %o3;                            \
        ld##s   [%o2], %g4;                     \
        add     %o2, %o4, %o2;                  \
        st##s   %g4, [%o1];                     \
        brnz,pt %o3, 2b;                        \
        add     %o1, n, %o1;                    \
1:

        .align 32
        ENTRY(ddi_rep_get8)
        ALTENTRY(ddi_mem_rep_get8)
        ldn      [%o0 + AHI_REP_GET8], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_get8)
        SET_SIZE(ddi_mem_rep_get8)

        .align 16
        ENTRY(i_ddi_rep_get8)
        DDI_REP_GET(1,ub)
        retl
        nop
        SET_SIZE(i_ddi_rep_get8)

        .align 32
        ENTRY(ddi_rep_get16)
        ALTENTRY(ddi_mem_rep_get16)
        ldn     [%o0 + AHI_REP_GET16], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_get16)
        SET_SIZE(ddi_mem_rep_get16)

        .align 16
        ENTRY(i_ddi_rep_get16)
        ALTENTRY(i_ddi_swap_rep_get16)
        DDI_REP_GET(2,uh)
        retl
        nop
        SET_SIZE(i_ddi_rep_get16)
        SET_SIZE(i_ddi_swap_rep_get16)

        .align 32
        ENTRY(ddi_rep_get32)
        ALTENTRY(ddi_mem_rep_get32)
        ldn      [%o0 + AHI_REP_GET32], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_get32)
        SET_SIZE(ddi_mem_rep_get32)

        .align 16
        ENTRY(i_ddi_rep_get32)
        ALTENTRY(i_ddi_swap_rep_get32)
        DDI_REP_GET(4,/**/)
        retl
        nop
        SET_SIZE(i_ddi_rep_get32)
        SET_SIZE(i_ddi_swap_rep_get32)

        .align 32
        ENTRY(ddi_rep_get64)
        ALTENTRY(ddi_mem_rep_get64)
        ldn      [%o0 + AHI_REP_GET64], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_get64)
        SET_SIZE(ddi_mem_rep_get64)

        .align 16
        ENTRY(i_ddi_rep_get64)
        ALTENTRY(i_ddi_swap_rep_get64)
        DDI_REP_GET(8,x)
        retl
        nop
        SET_SIZE(i_ddi_rep_get64)
        SET_SIZE(i_ddi_swap_rep_get64)

/*
 * This define processes the repetitive put functionality.  Automatic
 * incrementing of the device address is determined by the flag field
 * %o4.  If this is set for AUTOINCR, %o4 is updated with 1 for the
 * subsequent increment in 2:.
 *
 * If this flag is not set for AUTOINCR, %o4 is update with a value of 0 thus
 * making the increment operation a non-operation.
 */
#define DDI_REP_PUT(n,s)                        \
        cmp     DDI_DEV_NO_AUTOINCR, %o4;       \
        mov     %g0, %o4;                       \
        brz,pn  %o3, 1f;                        \
        movnz   %xcc, n, %o4;                   \
2:                                              \
        dec     %o3;                            \
        ld##s   [%o1], %g4;                     \
        add     %o1, n, %o1;                    \
        st##s   %g4, [%o2];                     \
        brnz,pt %o3, 2b;                        \
        add     %o2, %o4, %o2;                  \
1:

        .align 32
        ENTRY(ddi_rep_put8)
        ALTENTRY(ddi_mem_rep_put8)
        ldn      [%o0 + AHI_REP_PUT8], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_put8)
        SET_SIZE(ddi_mem_rep_put8)

        .align 16
        ENTRY(i_ddi_rep_put8)
        DDI_REP_PUT(1,ub)
        retl
        nop
        SET_SIZE(i_ddi_rep_put8)

        .align 32
        ENTRY(ddi_rep_put16)
        ALTENTRY(ddi_mem_rep_put16)
        ldn      [%o0 + AHI_REP_PUT16], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_put16)
        SET_SIZE(ddi_mem_rep_put16)

        .align 16
        ENTRY(i_ddi_rep_put16)
        ALTENTRY(i_ddi_swap_rep_put16)
        DDI_REP_PUT(2,uh)
        retl
        nop
        SET_SIZE(i_ddi_rep_put16)
        SET_SIZE(i_ddi_swap_rep_put16)

        .align 32
        ENTRY(ddi_rep_put32)
        ALTENTRY(ddi_mem_rep_put32)
        ldn      [%o0 + AHI_REP_PUT32], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_put32)
        SET_SIZE(ddi_mem_rep_put32)

        .align 16
        ENTRY(i_ddi_rep_put32)
        ALTENTRY(i_ddi_swap_rep_put32)
        DDI_REP_PUT(4,/**/)
        retl
        nop
        SET_SIZE(i_ddi_rep_put32)
        SET_SIZE(i_ddi_swap_rep_put32)

        .align 32
        ENTRY(ddi_rep_put64)
        ALTENTRY(ddi_mem_rep_put64)
        ldn      [%o0 + AHI_REP_PUT64], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_rep_put64)
        SET_SIZE(ddi_mem_rep_put64)

        .align 16
        ENTRY(i_ddi_rep_put64)
        ALTENTRY(i_ddi_swap_rep_put64)
        DDI_REP_PUT(8,x)
        retl
        nop
        SET_SIZE(i_ddi_rep_put64)
        SET_SIZE(i_ddi_swap_rep_put64)

        .align 16
        ENTRY(ddi_io_rep_get8)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_GET8], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_get8)

        .align 16
        ENTRY(ddi_io_rep_get16)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_GET16], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_get16)

        .align 16
        ENTRY(ddi_io_rep_get32)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_GET32], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_get32)

        .align 16
        ENTRY(ddi_io_rep_get64)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_GET64], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_get64)

        .align 64
        ENTRY(ddi_check_acc_handle)
        save    %sp, -SA(WINDOWSIZE), %sp       ! get a new window
        ldn     [%i0 + AHI_FAULT_CHECK], %g1
        jmpl    %g1, %o7
        mov     %i0, %o0
        brnz,a,pn %o0, 0f                       ! if (return_value != 0)
        mov     -1, %o0                         !       return (DDI_FAILURE)
0:                                              ! else  return (DDI_SUCCESS)
        sra     %o0, 0, %i0
        ret
        restore
        SET_SIZE(ddi_check_acc_handle)

        .align 16
        ENTRY(i_ddi_acc_fault_check)
        retl
        ld      [%o0 + AHI_FAULT], %o0
        SET_SIZE(i_ddi_acc_fault_check)

        .align 16
        ENTRY(ddi_io_rep_put8)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_PUT8], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_put8)

        .align 16
        ENTRY(ddi_io_rep_put16)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_PUT16], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_put16)

        .align 16
        ENTRY(ddi_io_rep_put32)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_PUT32], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_put32)

        .align 16
        ENTRY(ddi_io_rep_put64)
        set     DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
        ldn     [%o0 + AHI_REP_PUT64], %g1
        jmpl    %g1, %g0
        nop
        SET_SIZE(ddi_io_rep_put64)

        ENTRY(do_peek)
        rdpr    %pstate, %o3    ! check ints
        andcc   %o3, PSTATE_IE, %g0
        bz,a    done
        or      %g0, 1, %o0     ! Return failure if ints are disabled
        wrpr    %o3, PSTATE_IE, %pstate
        cmp     %o0, 8          ! 64-bit?
        bne,a   .peek_int
        cmp     %o0, 4          ! 32-bit?
        ldx     [%o1], %g1
        ba      .peekdone
        stx     %g1, [%o2]
.peek_int:
        bne,a   .peek_half
        cmp     %o0, 2          ! 16-bit?
        lduw    [%o1], %g1
        ba      .peekdone
        stuw    %g1, [%o2]
.peek_half:
        bne,a   .peek_byte
        ldub    [%o1], %g1      ! 8-bit!
        lduh    [%o1], %g1
        ba      .peekdone
        stuh    %g1, [%o2]
.peek_byte:
        stub    %g1, [%o2]
.peekdone:
        membar  #Sync           ! Make sure the loads take
        rdpr    %pstate, %o3    ! check&enable ints
        andcc   %o3, PSTATE_IE, %g0
        bnz     1f
        nop
        wrpr    %o3, PSTATE_IE, %pstate
1:
        mov     %g0, %o0
done:
        retl
        nop
        SET_SIZE(do_peek)

        ENTRY(do_poke)
        cmp     %o0, 8          ! 64 bit?
        bne,a   .poke_int
        cmp     %o0, 4          ! 32-bit?
        ldx     [%o2], %g1
        ba      .pokedone
        stx     %g1, [%o1]
.poke_int:
        bne,a   .poke_half
        cmp     %o0, 2          ! 16-bit?
        lduw    [%o2], %g1
        ba      .pokedone
        stuw    %g1, [%o1]
.poke_half:
        bne,a   .poke_byte
        ldub    [%o2], %g1      ! 8-bit!
        lduh    [%o2], %g1
        ba      .pokedone
        stuh    %g1, [%o1]
.poke_byte:
        stub    %g1, [%o1]
.pokedone:
        membar  #Sync
        retl
        mov     %g0, %o0
        SET_SIZE(do_poke)


/*
 * The peek_fault() and poke_fault() routines below are used as on_trap()
 * trampoline routines.  i_ddi_peek and i_ddi_poke execute do_peek and do_poke
 * under on_trap protection (see <sys/ontrap.h>), but modify ot_trampoline to
 * refer to the corresponding routine below.  If a trap occurs, the trap code
 * will bounce back to the trampoline code, which will effectively cause
 * do_peek or do_poke to return DDI_FAILURE, instead of longjmp'ing back to
 * on_trap.  In the case of a peek, we may also need to re-enable interrupts.
 */
        .seg    ".data"
.peek_panic:
        .asciz  "peek_fault: missing or invalid on_trap_data"
.poke_panic:
        .asciz  "poke_fault: missing or invalid on_trap_data"

        ENTRY(peek_fault)
        ldn     [THREAD_REG + T_ONTRAP], %o0    ! %o0 = on_trap_data pointer
        brz,pn  %o0, .peekfail                  ! if (%o0 == NULL) panic
        nop
        lduh    [%o0 + OT_PROT], %o1            ! %o1 = %o0->ot_prot
        andcc   %o1, OT_DATA_ACCESS, %g0        ! if (!(%o1 & OT_DATA_ACCESS))
        bz,pn   %icc, .peekfail                 !     panic
        rdpr    %pstate, %o3

        andcc   %o3, PSTATE_IE, %g0             ! enable interrupts
        bnz     1f
        nop
        wrpr    %o3, PSTATE_IE, %pstate
1:
        retl
        sub     %g0, 1, %o0                     ! return (DDI_FAILURE);
.peekfail:
        set     .peek_panic, %o0                ! Load panic message
        call    panic                           ! Panic if bad t_ontrap data
        nop
        SET_SIZE(peek_fault)


        ENTRY(poke_fault)
        ldn     [THREAD_REG + T_ONTRAP], %o0    ! %o0 = on_trap_data pointer
        brz,pn  %o0, .pokefail                  ! if (%o0 == NULL) panic
        nop
        lduh    [%o0 + OT_PROT], %o1            ! %o1 = %o0->ot_prot
        andcc   %o1, OT_DATA_ACCESS, %g0        ! if (!(%o1 & OT_DATA_ACCESS))
        bz,pn   %icc, .pokefail                 !     panic
        nop
        retl
        sub     %g0, 1, %o0                     ! return (DDI_FAILURE);
.pokefail:
        set     .poke_panic, %o0                ! Load panic message
        call    panic                           ! Panic if bad t_ontrap data
        nop
        SET_SIZE(poke_fault)


/*
 * IO Fault Services
 *
 * Support for protected IO accesses is implemented in the following
 * functions.  A driver may request one of three protection mechanisms
 * that enable the system to survive an access errors.  The protection
 * mechansim is set-up during ddi_regs_map_setup time and may be one of:
 *
 *      DDI_DEFAULT_ACC - no error protection requested.  We will
 *                      use the standard ddi_get/ddi_put operations
 *                      defined above.
 *
 *      DDI_FLAGERR - Driver requests that errors encountered will
 *                      be flagged by the system.  The driver is
 *                      responsible for checking the error status
 *                      of the access with a call to ddi_acc_err_get()
 *                      upon return of ddi_get or ddi_put.  To prevent
 *                      an access from causing a system we use internal
 *                      on_trap semantics.
 *
 *                      The system, depending upon the error,
 *                      may or may not panic.
 *
 *      DDI_CAUTIOUS_ACC - Driver expects that the access may cause
 *                      an error to occur.  The system will return
 *                      an error status but will not generate an ereport.
 *                      The system will also ensure synchronous and
 *                      exclusive access to the IO space accessed by
 *                      the caller.
 *
 *                      To prevent an access from causing a system panic,
 *                      we use on_trap semantics to catch the error and
 *                      set error status.
 *
 *      If a read access error is detected and DDI_CAUTIOUS_ACC or
 *      DDI_FLAGERR_ACC protection was requested, we will trampoline to the
 *      error handler, i_ddi_trampoline.  i_ddi_trampoline will:
 *              - check for proper protection semantics
 *              - set the error status of the access handle to DDI_FM_NONFATAL
 *              - re-enable interrupts if neccessary
 *              - longjmp back to the initiating access function.

 *      If a write access error is detected, an interrupt is typically
 *      generated and claimed by a bus nexus responsible for the write
 *      transaction.  The nexus error handler is expected to set the
 *      error status and the IO initiating driver is expected to check
 *      for a failed transaction via ddi_fm_acc_err_get().
 *
 */

        .seg    ".data"
.acc_panic:
        .asciz  "DDI access: missing or invalid on_trap_data"

        ENTRY(i_ddi_caut_trampoline)
        ldn     [THREAD_REG + T_ONTRAP], %o5    ! %o5 = curthread->t_ontrap
        lduh    [%o5 + OT_PROT], %o1            ! %o1 = %o0->ot_prot
        andcc   %o1, OT_DATA_ACCESS, %g0        ! if (!(%o1 & OT_DATA_ACCESS))
        bz,pn   %icc, .cautaccfail              !     panic
        rdpr    %pstate, %o3
        andcc   %o3, PSTATE_IE, %g0             ! enable interrupts
        bnz     1f
        nop
        wrpr    %o3, PSTATE_IE, %pstate
1:
        ldn     [%o5 + OT_HANDLE], %o0          ! %o0 = ot_handle
        brz,pn  %o0, .cautaccfail               ! if (ot_handle == NULL) panic
        nop
        ldn     [%o0 + AHI_ERR], %o4            ! %o4 = hp->ahi_err
        membar  #Sync
        stx     %g0, [%o4 + ERR_ENA]            ! ahi_err->err_ena = 0
        mov     -2, %o0
        st      %o0, [%o4 + ERR_STATUS]         ! ahi_err->err_status = NONFATAL
        b       longjmp                         ! longjmp back
        add     %o5, OT_JMPBUF, %o0             ! %o0 = &ot_jmpbuf
.cautaccfail:
        set     .acc_panic, %o0                 ! Load panic message
        call    panic                           ! Panic if bad t_ontrap data
        nop
        SET_SIZE(i_ddi_caut_trampoline)

/*
 * DDI on_trap set-up functions,  i_ddi_ontrap() and i_ddinotrap() are used
 * to protect * ddi_get accesses for DDI_CAUT_ACC.  i_ddi_ontrap() sets
 * the jumpbuf (setjmp) that will return back to the access routine from
 * i_ddi_trampoline().  DDI_NOPROTECT() clears the ontrap set-up.
 */
        ENTRY(i_ddi_ontrap)
        ldn     [%o0 + AHI_ERR], %o4
        ldn     [%o4 + ERR_ONTRAP],  %o4        ! %o4 = hp->ahi_err->err_ontrap
        ldn     [THREAD_REG + T_ONTRAP], %o5    ! %o5 = curthread->t_ontrap
        stn     %o5, [%o4 + OT_PREV]            ! ot_prev = t_ontrap
        membar  #Sync                           ! force error barrier
        stn     %o4, [THREAD_REG + T_ONTRAP]    ! t_ontrap = err_ontrap
        b       setjmp
        add     %o4, OT_JMPBUF, %o0
        SET_SIZE(i_ddi_ontrap)

        ENTRY(i_ddi_notrap)
        membar  #Sync                           ! force error barrier
        ldn     [%o0 + AHI_ERR], %o4
        ldn     [%o4 + ERR_ONTRAP],  %o4        ! %o4 = hp->ahi_err->err_ontrap
        ldn     [%o4 + OT_PREV], %o4
        retl
        stn     %o4, [THREAD_REG + T_ONTRAP]    ! restore curthread->t_ontrap
        SET_SIZE(i_ddi_notrap)

/*
 * Internal on_trap set-up macros.  DDI_PROTECT() and DDI_NOPROTECT() are used
 * to protect * ddi_get accesses for DDI_FLAGERR_ACC.  DDI_NOPROTECT() sets
 * the jumpbuf that will return back to the access routine from
 * i_ddi_protect_trampoline().  DDI_NOPROTECT() clears the ontrap set-up.
 */
        ENTRY(i_ddi_prot_trampoline)
        ldn     [THREAD_REG + T_ONTRAP], %o5    ! %o5 = curthread->t_ontrap
        lduh    [%o5 + OT_PROT], %o1            ! %o1 = %o0->ot_prot
        andcc   %o1, OT_DATA_ACCESS, %g0        ! if (!(%o1 & OT_DATA_ACCESS))
        bz,pn   %icc, .protaccfail              !     panic
        rdpr    %pstate, %o3
        andcc   %o3, PSTATE_IE, %g0             ! enable interrupts
        bnz     1f
        nop
        wrpr    %o3, PSTATE_IE, %pstate
1:
        ldn     [%o5 + OT_HANDLE], %o0          ! %o0 = ot_handle
        brz,pn  %o0, .protaccfail               ! if (ot_handle == NULL) panic
        nop
        ldn     [%o0 + AHI_ERR], %o4            ! %o4 = hp->ahi_err
        stn     %g0, [%o4 + ERR_ENA]            ! ahi_err->err_ena = 0
        mov     -2, %o0
        st      %o0, [%o4 + ERR_STATUS]         ! ahi_err->err_status = NONFATAL
        ldn     [%o5 + OT_PREV], %o0            ! restore ontrap
        membar  #Sync                           ! force error barrier
        stn     %o0, [THREAD_REG + T_ONTRAP];
        b       longjmp                         ! longjmp back
        add     %o5, OT_JMPBUF, %o0             ! %o0 = &ot_jmpbuf
.protaccfail:
        set     .acc_panic, %o0                 ! Load panic message
        call    panic                           ! Panic if bad t_ontrap data
        nop
        SET_SIZE(i_ddi_prot_trampoline)

#define DDI_PROTECT()                           \
        ldn     [%o0 + AHI_ERR], %o4;           \
        ldn     [%o4 + ERR_ONTRAP],  %o4;       \
        ldn     [THREAD_REG + T_ONTRAP], %o5;   \
        stn     %o5, [%o4 + OT_PREV];           \
        membar  #Sync;                          \
        stn     %o4, [THREAD_REG + T_ONTRAP];   \
        add     %o4, OT_JMPBUF, %o0;            \
        stn     %o7, [%o0 + L_PC];              \
        stn     %sp, [%o0 + L_SP];              \
        clr     %o0;

#define DDI_NOPROTECT()                         \
        ldn     [THREAD_REG + T_ONTRAP], %o4;   \
        ldn     [%o4 + OT_PREV], %o5;           \
        membar  #Sync;                          \
        stn     %o5, [THREAD_REG + T_ONTRAP];

/*
 * DDI_FLAGERR_ACC specific get/put routines.
 */
        .align 16
        ENTRY(i_ddi_prot_get8)
        DDI_PROTECT()                           ! set ontrap protection
        ldub    [%o1], %o2                      ! do the io access
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        mov     %o2, %o0                        ! set return value
        SET_SIZE(i_ddi_prot_get8)

        .align 16
        ENTRY(i_ddi_prot_get16)
        DDI_PROTECT()                           ! set ontrap protection
        lduh    [%o1], %o2                      ! do the io access
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        mov     %o2, %o0                        ! set return value
        SET_SIZE(i_ddi_prot_get16)

        .align 16
        ENTRY(i_ddi_prot_get32)
        DDI_PROTECT()                           ! set ontrap protection
        ld      [%o1], %o2                      ! do the io access
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        mov     %o2, %o0                        ! set return value
        SET_SIZE(i_ddi_prot_get32)

        .align 16
        ENTRY(i_ddi_prot_get64)
        DDI_PROTECT()                           ! set ontrap protection
        ldx     [%o1], %o2                      ! do the io access
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        mov     %o2, %o0                        ! set return value
        SET_SIZE(i_ddi_prot_get64)

        .align 16
        ENTRY(i_ddi_prot_put8)
        stub    %o2, [%o1]                      ! do the io access
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_put8)

        .align 16
        ENTRY(i_ddi_prot_put16)
        stuh    %o2, [%o1]                      ! do the io access
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_put16)

        .align 16
        ENTRY(i_ddi_prot_put32)
        st      %o2, [%o1]                      ! do the io access
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_put32)

        .align 16
        ENTRY(i_ddi_prot_put64)
        stx     %o2, [%o1]                      ! do the io access
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_put64)

        .align 16
        ENTRY(i_ddi_prot_rep_get8)
        DDI_PROTECT()                           ! set ontrap protection
        tst     %o0                             ! check access error
        bnz,a   1f
        nop
        DDI_REP_GET(1,ub)
1:
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        nop
        SET_SIZE(i_ddi_prot_rep_get8)

        .align 16
        ENTRY(i_ddi_prot_rep_get16)
        DDI_PROTECT()                           ! set ontrap protection
        tst     %o0                             ! check access error
        bnz,a   1f
        nop
        DDI_REP_GET(2,uh)
1:
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        nop
        SET_SIZE(i_ddi_prot_rep_get16)

        .align 16
        ENTRY(i_ddi_prot_rep_get32)
        DDI_PROTECT()                           ! set ontrap protection
        tst     %o0                             ! check access error
        bnz,a   1f
        nop
        DDI_REP_GET(4,/**/)
1:
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        nop
        SET_SIZE(i_ddi_prot_rep_get32)

        .align 16
        ENTRY(i_ddi_prot_rep_get64)
        DDI_PROTECT()                           ! set ontrap protection
        tst     %o0                             ! check access error
        bnz,a   1f
        nop
        DDI_REP_GET(8,x)
1:
        DDI_NOPROTECT()                         ! remove protection & ret
        retl
        nop
        SET_SIZE(i_ddi_prot_rep_get64)

        .align 16
        ENTRY(i_ddi_prot_rep_put8)
        DDI_REP_PUT(1,ub)
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_rep_put8)

        .align 16
        ENTRY(i_ddi_prot_rep_put16)
        DDI_REP_PUT(2,uh)
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_rep_put16)

        .align 16
        ENTRY(i_ddi_prot_rep_put32)
        DDI_REP_PUT(4,/**/)
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_rep_put32)

        .align 16
        ENTRY(i_ddi_prot_rep_put64)
        DDI_REP_PUT(8,x)
        retl
        membar  #Sync;
        SET_SIZE(i_ddi_prot_rep_put64)

/*
 * Common DDI_CAUTIOUS_ACC routine called from cautious access routines
 * in ddi_impl.c
 */
        ENTRY(i_ddi_caut_get)
        rdpr    %pstate, %o3    ! check ints
        andcc   %o3, PSTATE_IE, %g0
        bz,a    cautdone
        nop
        wrpr    %o3, PSTATE_IE, %pstate
        cmp     %o0, 8          ! 64-bit?
        bne,a   .get_int
        cmp     %o0, 4          ! 32-bit?
        ldx     [%o1], %g1
        ba      .getdone
        stx     %g1, [%o2]
.get_int:
        bne,a   .get_half
        cmp     %o0, 2          ! 16-bit?
        lduw    [%o1], %g1
        ba      .getdone
        stuw    %g1, [%o2]
.get_half:
        bne,a   .get_byte
        ldub    [%o1], %g1      ! 8-bit!
        lduh    [%o1], %g1
        ba      .getdone
        stuh    %g1, [%o2]
.get_byte:
        stub    %g1, [%o2]
.getdone:
        rdpr    %pstate, %o3    ! check&enable ints
        andcc   %o3, PSTATE_IE, %g0
        bnz,a   cautdone
        nop
        wrpr    %o3, PSTATE_IE, %pstate
cautdone:
        retl
        nop
        SET_SIZE(i_ddi_caut_get)