/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include "assym.h"

/*
 * General assembly language routines.
 * It is the intent of this file to contain routines that are
 * specific to cpu architecture.
 */

/*
 * WARNING: If you add a fast trap handler which can be invoked by a
 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
 * instead of "done" instruction to return back to the user mode. See
 * comments for the "fast_trap_done" entry point for more information.
 */
#define FAST_TRAP_DONE  \
        ba,a    fast_trap_done

#include <sys/machclock.h>
#include <sys/clock.h>


#include <sys/asm_linkage.h>
#include <sys/privregs.h>
#include <vm/hat_sfmmu.h>
#include <sys/machparam.h>      /* To get SYSBASE and PAGESIZE */
#include <sys/machthread.h>
#include <sys/clock.h>
#include <sys/intreg.h>
#include <sys/psr_compat.h>
#include <sys/isa_defs.h>
#include <sys/dditypes.h>
#include <sys/intr.h>
#include <sys/hypervisor_api.h>

#include "assym.h"

#define ICACHE_FLUSHSZ  0x20

/*
 * Softint generated when counter field of tick reg matches value field
 * of tick_cmpr reg
 */
        ENTRY_NP(tickcmpr_set)
        ! get 64-bit clock_cycles interval
        mov     %o0, %o2
        mov     8, %o3                  ! A reasonable initial step size
1:
        WR_TICKCMPR(%o2,%o4,%o5,__LINE__)       ! Write to TICK_CMPR

        GET_NATIVE_TIME(%o0,%o4,%o5,__LINE__)   ! Read %tick to confirm the
                                                ! value we wrote was in the
                                                ! future.

        cmp     %o2, %o0                ! If the value we wrote was in the
        bg,pt   %xcc, 2f                !   future, then blow out of here.
          sllx  %o3, 1, %o3             ! If not, then double our step size,
        ba,pt   %xcc, 1b                !   and take another lap.
          add   %o0, %o3, %o2           !
2:
        retl
          nop
        SET_SIZE(tickcmpr_set)

        ENTRY_NP(tickcmpr_disable)
        mov     1, %g1
        sllx    %g1, TICKINT_DIS_SHFT, %o0
        WR_TICKCMPR(%o0,%o4,%o5,__LINE__)       ! Write to TICK_CMPR
        retl
          nop
        SET_SIZE(tickcmpr_disable)

        .seg    ".text"
tick_write_delta_panic:
        .asciz  "tick_write_delta: not supported, delta: 0x%lx"

/*
 * tick_write_delta() is intended to increment %stick by the specified delta,
 * but %stick is only writeable in hyperprivileged mode and at present there
 * is no provision for this. tick_write_delta is called by the cylic subsystem
 * if a negative %stick delta is observed after cyclic processing is resumed
 * after an event such as an OS suspend/resume. On sun4v, the suspend/resume
 * routines should adjust the %stick offset preventing the cyclic subsystem
 * from detecting a negative delta. If a negative delta is detected, panic the
 * system. The negative delta could be caused by improper %stick
 * synchronization after a suspend/resume.
 */
        ENTRY_NP(tick_write_delta)
        sethi   %hi(tick_write_delta_panic), %o1
        save    %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller
        mov     %i0, %o1
        call    panic
          or    %i1, %lo(tick_write_delta_panic), %o0
        /*NOTREACHED*/
        retl
          nop

        ENTRY_NP(tickcmpr_disabled)
        RD_TICKCMPR(%g1,%o0,%o1,__LINE__)
        retl
          srlx  %g1, TICKINT_DIS_SHFT, %o0
        SET_SIZE(tickcmpr_disabled)

/*
 * Get current tick
 */

        ENTRY(gettick)
        ALTENTRY(randtick)
        GET_NATIVE_TIME(%o0,%o2,%o3,__LINE__)
        retl
          nop
        SET_SIZE(randtick)
        SET_SIZE(gettick)

/*
 * Get current tick. For trapstat use only.
 */
        ENTRY(rdtick)
        retl
        RD_TICK_PHYSICAL(%o0)
        SET_SIZE(rdtick)


/*
 * Return the counter portion of the tick register.
 */

        ENTRY_NP(gettick_counter)
        RD_TICK(%o0,%o1,%o2,__LINE__)
        retl
        nop
        SET_SIZE(gettick_counter)

        ENTRY_NP(gettick_npt)
        RD_TICK_PHYSICAL(%o0)
        retl
        srlx    %o0, 63, %o0
        SET_SIZE(gettick_npt)

        ENTRY_NP(getstick_npt)
        RD_STICK_PHYSICAL(%o0)
        retl
        srlx    %o0, 63, %o0
        SET_SIZE(getstick_npt)

/*
 * Provide a C callable interface to the trap that reads the hi-res timer.
 * Returns 64-bit nanosecond timestamp in %o0 and %o1.
 */

        ENTRY_NP(gethrtime)
        GET_HRTIME(%g1,%o0,%o1,%o2,%o3,%o4,%o5,%g2,__LINE__)
                                                        ! %g1 = hrtime
        retl
          mov   %g1, %o0
        SET_SIZE(gethrtime)

        ENTRY_NP(gethrtime_unscaled)
        GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__)   ! %g1 = native time
        retl
          mov   %g1, %o0
        SET_SIZE(gethrtime_unscaled)

        ENTRY_NP(gethrtime_waitfree)
        ALTENTRY(dtrace_gethrtime)
        GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__)   ! %g1 = native time
        NATIVE_TIME_TO_NSEC(%g1, %o2, %o3)
        retl
          mov   %g1, %o0
        SET_SIZE(dtrace_gethrtime)
        SET_SIZE(gethrtime_waitfree)

        ENTRY(gethrtime_max)
        NATIVE_TIME_MAX(%g1)
        NATIVE_TIME_TO_NSEC(%g1, %o0, %o1)

        ! hrtime_t's are signed, max hrtime_t must be positive
        mov     -1, %o2
        brlz,a  %g1, 1f
          srlx  %o2, 1, %g1
1:
        retl
          mov   %g1, %o0
        SET_SIZE(gethrtime_max)

        ENTRY(scalehrtime)
        ldx     [%o0], %o1
        NATIVE_TIME_TO_NSEC(%o1, %o2, %o3)
        retl
          stx   %o1, [%o0]
        SET_SIZE(scalehrtime)

/*
 * Fast trap to return a timestamp, uses trap window, leaves traps
 * disabled.  Returns a 64-bit nanosecond timestamp in %o0 and %o1.
 *
 * This is the handler for the ST_GETHRTIME trap.
 */

        ENTRY_NP(get_timestamp)
        GET_HRTIME(%g1,%g2,%g3,%g4,%g5,%o0,%o1,%o2,__LINE__)
        ! %g1 = hrtime
        srlx    %g1, 32, %o0                            ! %o0 = hi32(%g1)
        srl     %g1, 0, %o1                             ! %o1 = lo32(%g1)
        FAST_TRAP_DONE
        SET_SIZE(get_timestamp)

/*
 * Macro to convert GET_HRESTIME() bits into a timestamp.
 *
 * We use two separate macros so that the platform-dependent GET_HRESTIME()
 * can be as small as possible; CONV_HRESTIME() implements the generic part.
 */
#define CONV_HRESTIME(hrestsec, hrestnsec, adj, nslt, nano) \
        brz,pt  adj, 3f;                /* no adjustments, it's easy */ \
        add     hrestnsec, nslt, hrestnsec; /* hrest.tv_nsec += nslt */ \
        brlz,pn adj, 2f;                /* if hrestime_adj negative */  \
          srlx  nslt, ADJ_SHIFT, nslt;  /* delay: nslt >>= 4 */         \
        subcc   adj, nslt, %g0;         /* hrestime_adj - nslt/16 */    \
        movg    %xcc, nslt, adj;        /* adj by min(adj, nslt/16) */  \
        ba      3f;                     /* go convert to sec/nsec */    \
          add   hrestnsec, adj, hrestnsec; /* delay: apply adjustment */ \
2:      addcc   adj, nslt, %g0;         /* hrestime_adj + nslt/16 */    \
        bge,a,pt %xcc, 3f;              /* is adj less negative? */     \
          add   hrestnsec, adj, hrestnsec; /* yes: hrest.nsec += adj */ \
        sub     hrestnsec, nslt, hrestnsec; /* no: hrest.nsec -= nslt/16 */ \
3:      cmp     hrestnsec, nano;        /* more than a billion? */      \
        bl,pt   %xcc, 4f;               /* if not, we're done */        \
          nop;                          /* delay: do nothing :( */      \
        add     hrestsec, 1, hrestsec;  /* hrest.tv_sec++; */           \
        sub     hrestnsec, nano, hrestnsec; /* hrest.tv_nsec -= NANOSEC; */     \
        ba,a    3b;                     /* check >= billion again */    \
4:

        ENTRY_NP(gethrestime)
        GET_HRESTIME(%o1,%o2,%o3,%o4,%o5,%g1,%g2,%g3,%g4,__LINE__)
        CONV_HRESTIME(%o1, %o2, %o3, %o4, %o5)
        stn     %o1, [%o0]
        retl
          stn   %o2, [%o0 + CLONGSIZE]
        SET_SIZE(gethrestime)

/*
 * Similar to gethrestime(), but gethrestime_sec() returns current hrestime
 * seconds.
 */
        ENTRY_NP(gethrestime_sec)
        GET_HRESTIME(%o0,%o2,%o3,%o4,%o5,%g1,%g2,%g3,%g4,__LINE__)
        CONV_HRESTIME(%o0, %o2, %o3, %o4, %o5)
        retl                                    ! %o0 current hrestime seconds
          nop
        SET_SIZE(gethrestime_sec)

/*
 * Returns the hrestime on the last tick.  This is simpler than gethrestime()
 * and gethrestime_sec():  no conversion is required.  gethrestime_lasttick()
 * follows the same locking algorithm as GET_HRESTIME and GET_HRTIME,
 * outlined in detail in clock.h.  (Unlike GET_HRESTIME/GET_HRTIME, we don't
 * rely on load dependencies to effect the membar #LoadLoad, instead declaring
 * it explicitly.)
 */
        ENTRY_NP(gethrestime_lasttick)
        sethi   %hi(hres_lock), %o1
0:
        lduw    [%o1 + %lo(hres_lock)], %o2     ! Load lock value
        membar  #LoadLoad                       ! Load of lock must complete
        andn    %o2, 1, %o2                     ! Mask off lowest bit
        ldn     [%o1 + %lo(hrestime)], %g1      ! Seconds.
        add     %o1, %lo(hrestime), %o4
        ldn     [%o4 + CLONGSIZE], %g2          ! Nanoseconds.
        membar  #LoadLoad                       ! All loads must complete
        lduw    [%o1 + %lo(hres_lock)], %o3     ! Reload lock value
        cmp     %o3, %o2                        ! If lock is locked or has
        bne     0b                              !   changed, retry.
          stn   %g1, [%o0]                      ! Delay: store seconds
        retl
          stn   %g2, [%o0 + CLONGSIZE]          ! Delay: store nanoseconds
        SET_SIZE(gethrestime_lasttick)

/*
 * Fast trap for gettimeofday().  Returns a timestruc_t in %o0 and %o1.
 *
 * This is the handler for the ST_GETHRESTIME trap.
 */

        ENTRY_NP(get_hrestime)
        GET_HRESTIME(%o0,%o1,%g1,%g2,%g3,%g4,%g5,%o2,%o3,__LINE__)
        CONV_HRESTIME(%o0, %o1, %g1, %g2, %g3)
        FAST_TRAP_DONE
        SET_SIZE(get_hrestime)

/*
 * Fast trap to return lwp virtual time, uses trap window, leaves traps
 * disabled.  Returns a 64-bit number in %o0:%o1, which is the number
 * of nanoseconds consumed.
 *
 * This is the handler for the ST_GETHRVTIME trap.
 *
 * Register usage:
 *      %o0, %o1 = return lwp virtual time
 *      %o2 = CPU/thread
 *      %o3 = lwp
 *      %g1 = scratch
 *      %g5 = scratch
 */
        ENTRY_NP(get_virtime)
        GET_NATIVE_TIME(%g5,%g1,%g2,__LINE__)   ! %g5 = native time in ticks
        CPU_ADDR(%g2, %g3)                      ! CPU struct ptr to %g2
        ldn     [%g2 + CPU_THREAD], %g2         ! thread pointer to %g2
        ldn     [%g2 + T_LWP], %g3              ! lwp pointer to %g3

        /*
         * Subtract start time of current microstate from time
         * of day to get increment for lwp virtual time.
         */
        ldx     [%g3 + LWP_STATE_START], %g1    ! ms_state_start
        sub     %g5, %g1, %g5

        /*
         * Add current value of ms_acct[LMS_USER]
         */
        ldx     [%g3 + LWP_ACCT_USER], %g1      ! ms_acct[LMS_USER]
        add     %g5, %g1, %g5
        NATIVE_TIME_TO_NSEC(%g5, %g1, %o0)

        srl     %g5, 0, %o1                     ! %o1 = lo32(%g5)
        srlx    %g5, 32, %o0                    ! %o0 = hi32(%g5)

        FAST_TRAP_DONE
        SET_SIZE(get_virtime)



        .seg    ".text"
hrtime_base_panic:
        .asciz  "hrtime_base stepping back"


        ENTRY_NP(hres_tick)
        save    %sp, -SA(MINFRAME), %sp ! get a new window

        sethi   %hi(hrestime), %l4
        ldstub  [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5  ! try locking
7:      tst     %l5
        bz,pt   %xcc, 8f                        ! if we got it, drive on
          ld    [%l4 + %lo(nsec_scale)], %l5    ! delay: %l5 = scaling factor
        ldub    [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5
9:      tst     %l5
        bz,a,pn %xcc, 7b
          ldstub        [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5
        ba,pt   %xcc, 9b
          ldub  [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5
8:
        membar  #StoreLoad|#StoreStore

        !
        ! update hres_last_tick.  %l5 has the scaling factor (nsec_scale).
        !
        ldx     [%l4 + %lo(hrtime_base)], %g1   ! load current hrtime_base
        GET_NATIVE_TIME(%l0,%l3,%l6,__LINE__)   ! current native time
        stx     %l0, [%l4 + %lo(hres_last_tick)]! prev = current
        ! convert native time to nsecs
        NATIVE_TIME_TO_NSEC_SCALE(%l0, %l5, %l2, NSEC_SHIFT)

        sub     %l0, %g1, %i1                   ! get accurate nsec delta

        ldx     [%l4 + %lo(hrtime_base)], %l1
        cmp     %l1, %l0
        bg,pn   %xcc, 9f
          nop

        stx     %l0, [%l4 + %lo(hrtime_base)]   ! update hrtime_base

        !
        ! apply adjustment, if any
        !
        ldx     [%l4 + %lo(hrestime_adj)], %l0  ! %l0 = hrestime_adj
        brz     %l0, 2f
                                                ! hrestime_adj == 0 ?
                                                ! yes, skip adjustments
          clr   %l5                             ! delay: set adj to zero
        tst     %l0                             ! is hrestime_adj >= 0 ?
        bge,pt  %xcc, 1f                        ! yes, go handle positive case
          srl   %i1, ADJ_SHIFT, %l5             ! delay: %l5 = adj

        addcc   %l0, %l5, %g0                   ! hrestime_adj < -adj ?
        bl,pt   %xcc, 2f                        ! yes, use current adj
          neg   %l5                             ! delay: %l5 = -adj
        ba,pt   %xcc, 2f
          mov   %l0, %l5                        ! no, so set adj = hrestime_adj
1:
        subcc   %l0, %l5, %g0                   ! hrestime_adj < adj ?
        bl,a,pt %xcc, 2f                        ! yes, set adj = hrestime_adj
          mov   %l0, %l5                        ! delay: adj = hrestime_adj
2:
        ldx     [%l4 + %lo(timedelta)], %l0     ! %l0 = timedelta
        sub     %l0, %l5, %l0                   ! timedelta -= adj

        stx     %l0, [%l4 + %lo(timedelta)]     ! store new timedelta
        stx     %l0, [%l4 + %lo(hrestime_adj)]  ! hrestime_adj = timedelta

        or      %l4, %lo(hrestime), %l2
        ldn     [%l2], %i2                      ! %i2:%i3 = hrestime sec:nsec
        ldn     [%l2 + CLONGSIZE], %i3
        add     %i3, %l5, %i3                   ! hrestime.nsec += adj
        add     %i3, %i1, %i3                   ! hrestime.nsec += nslt

        set     NANOSEC, %l5                    ! %l5 = NANOSEC
        cmp     %i3, %l5
        bl,pt   %xcc, 5f                        ! if hrestime.tv_nsec < NANOSEC
          sethi %hi(one_sec), %i1               ! delay
        add     %i2, 0x1, %i2                   ! hrestime.tv_sec++
        sub     %i3, %l5, %i3                   ! hrestime.tv_nsec - NANOSEC
        mov     0x1, %l5
        st      %l5, [%i1 + %lo(one_sec)]
5:
        stn     %i2, [%l2]
        stn     %i3, [%l2 + CLONGSIZE]          ! store the new hrestime

        membar  #StoreStore

        ld      [%l4 + %lo(hres_lock)], %i1
        inc     %i1                             ! release lock
        st      %i1, [%l4 + %lo(hres_lock)]     ! clear hres_lock

        ret
        restore

9:
        !
        ! release hres_lock
        !
        ld      [%l4 + %lo(hres_lock)], %i1
        inc     %i1
        st      %i1, [%l4 + %lo(hres_lock)]

        sethi   %hi(hrtime_base_panic), %o0
        call    panic
          or    %o0, %lo(hrtime_base_panic), %o0

        SET_SIZE(hres_tick)

        .seg    ".text"
kstat_q_panic_msg:
        .asciz  "kstat_q_exit: qlen == 0"

        ENTRY(kstat_q_panic)
        save    %sp, -SA(MINFRAME), %sp
        sethi   %hi(kstat_q_panic_msg), %o0
        call    panic
          or    %o0, %lo(kstat_q_panic_msg), %o0
        /*NOTREACHED*/
        SET_SIZE(kstat_q_panic)

#define BRZPN   brz,pn
#define BRZPT   brz,pt

#define KSTAT_Q_UPDATE(QOP, QBR, QZERO, QRETURN, QTYPE) \
        ld      [%o0 + QTYPE##CNT], %o1;        /* %o1 = old qlen */    \
        QOP     %o1, 1, %o2;                    /* %o2 = new qlen */    \
        QBR     %o1, QZERO;                     /* done if qlen == 0 */ \
        st      %o2, [%o0 + QTYPE##CNT];        /* delay: save qlen */  \
        ldx     [%o0 + QTYPE##LASTUPDATE], %o3;                 \
        ldx     [%o0 + QTYPE##TIME], %o4;       /* %o4 = old time */    \
        ldx     [%o0 + QTYPE##LENTIME], %o5;    /* %o5 = old lentime */ \
        sub     %g1, %o3, %o2;                  /* %o2 = time delta */  \
        mulx    %o1, %o2, %o3;                  /* %o3 = cur lentime */ \
        add     %o4, %o2, %o4;                  /* %o4 = new time */    \
        add     %o5, %o3, %o5;                  /* %o5 = new lentime */ \
        stx     %o4, [%o0 + QTYPE##TIME];       /* save time */         \
        stx     %o5, [%o0 + QTYPE##LENTIME];    /* save lentime */      \
QRETURN;                                                                \
        stx     %g1, [%o0 + QTYPE##LASTUPDATE]; /* lastupdate = now */

#if !defined(DEBUG)
/*
 * same as KSTAT_Q_UPDATE but without:
 * QBR     %o1, QZERO;
 * to be used only with non-debug build. mimics ASSERT() behaviour.
 */
#define KSTAT_Q_UPDATE_ND(QOP, QRETURN, QTYPE) \
        ld      [%o0 + QTYPE##CNT], %o1;        /* %o1 = old qlen */    \
        QOP     %o1, 1, %o2;                    /* %o2 = new qlen */    \
        st      %o2, [%o0 + QTYPE##CNT];        /* delay: save qlen */  \
        ldx     [%o0 + QTYPE##LASTUPDATE], %o3;                 \
        ldx     [%o0 + QTYPE##TIME], %o4;       /* %o4 = old time */    \
        ldx     [%o0 + QTYPE##LENTIME], %o5;    /* %o5 = old lentime */ \
        sub     %g1, %o3, %o2;                  /* %o2 = time delta */  \
        mulx    %o1, %o2, %o3;                  /* %o3 = cur lentime */ \
        add     %o4, %o2, %o4;                  /* %o4 = new time */    \
        add     %o5, %o3, %o5;                  /* %o5 = new lentime */ \
        stx     %o4, [%o0 + QTYPE##TIME];       /* save time */         \
        stx     %o5, [%o0 + QTYPE##LENTIME];    /* save lentime */      \
QRETURN;                                                                \
        stx     %g1, [%o0 + QTYPE##LASTUPDATE]; /* lastupdate = now */
#endif

        .align 16
        ENTRY(kstat_waitq_enter)
        GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
        KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
        SET_SIZE(kstat_waitq_enter)

        .align 16
        ENTRY(kstat_waitq_exit)
        GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
#if defined(DEBUG)
        KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, retl, KSTAT_IO_W)
#else
        KSTAT_Q_UPDATE_ND(sub, retl, KSTAT_IO_W)
#endif
        SET_SIZE(kstat_waitq_exit)

        .align 16
        ENTRY(kstat_runq_enter)
        GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
        KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
        SET_SIZE(kstat_runq_enter)

        .align 16
        ENTRY(kstat_runq_exit)
        GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
#if defined(DEBUG)
        KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, retl, KSTAT_IO_R)
#else
        KSTAT_Q_UPDATE_ND(sub, retl, KSTAT_IO_R)
#endif
        SET_SIZE(kstat_runq_exit)

        .align 16
        ENTRY(kstat_waitq_to_runq)
        GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
#if defined(DEBUG)
        KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_W)
#else
        KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_W)
#endif
        KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
        SET_SIZE(kstat_waitq_to_runq)

        .align 16
        ENTRY(kstat_runq_back_to_waitq)
        GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
#if defined(DEBUG)
        KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_R)
#else
        KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_R)
#endif
        KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
        SET_SIZE(kstat_runq_back_to_waitq)

        /*
         *  -- WARNING --
         *
         * The following variables MUST be together on a 128-byte boundary.
         * In addition to the primary performance motivation (having them all
         * on the same cache line(s)), code here and in the GET*TIME() macros
         * assumes that they all have the same high 22 address bits (so
         * there's only one sethi).
         */
        .seg    ".data"
        .global timedelta, hres_last_tick, hrestime, hrestime_adj
        .global hres_lock, nsec_scale, hrtime_base, traptrace_use_stick
        .global nsec_shift, adj_shift, native_tick_offset, native_stick_offset

        /* XXX - above comment claims 128-bytes is necessary */
        .align  64
timedelta:
        .word   0, 0            /* int64_t */
hres_last_tick:
        .word   0, 0            /* hrtime_t */
hrestime:
        .nword  0, 0            /* 2 longs */
hrestime_adj:
        .word   0, 0            /* int64_t */
hres_lock:
        .word   0
nsec_scale:
        .word   0
hrtime_base:
        .word   0, 0
traptrace_use_stick:
        .word   0
nsec_shift:
        .word   NSEC_SHIFT
adj_shift:
        .word   ADJ_SHIFT
        .align  8
native_tick_offset:
        .word   0, 0
        .align  8
native_stick_offset:
        .word   0, 0


/*
 * drv_usecwait(clock_t n)      [DDI/DKI - section 9F]
 * usec_delay(int n)            [compatibility - should go one day]
 * Delay by spinning.
 *
 * delay for n microseconds.  numbers <= 0 delay 1 usec
 *
 * With UltraSPARC-III the combination of supporting mixed-speed CPUs
 * and variable clock rate for power management requires that we
 * use %stick to implement this routine.
 */

        ENTRY(drv_usecwait)
        ALTENTRY(usec_delay)
        brlez,a,pn %o0, 0f
          mov   1, %o0
0:
        sethi   %hi(sticks_per_usec), %o1
        lduw    [%o1 + %lo(sticks_per_usec)], %o1
        mulx    %o1, %o0, %o1           ! Scale usec to ticks
        inc     %o1                     ! We don't start on a tick edge
        GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
        add     %o1, %o2, %o1

1:      cmp     %o1, %o2
        GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
        bgeu,pt %xcc, 1b
          nop
        retl
          nop
        SET_SIZE(usec_delay)
        SET_SIZE(drv_usecwait)

/*
 * Level-14 interrupt prologue.
 */
        ENTRY_NP(pil14_interrupt)
        CPU_ADDR(%g1, %g2)
        rdpr    %pil, %g6                       ! %g6 = interrupted PIL
        stn     %g6, [%g1 + CPU_PROFILE_PIL]    ! record interrupted PIL
        rdpr    %tstate, %g6
        rdpr    %tpc, %g5
        btst    TSTATE_PRIV, %g6                ! trap from supervisor mode?
        bnz,a,pt %xcc, 1f
          stn   %g5, [%g1 + CPU_PROFILE_PC]     ! if so, record kernel PC
        stn     %g5, [%g1 + CPU_PROFILE_UPC]    ! if not, record user PC
        ba      pil_interrupt_common            ! must be large-disp branch
          stn   %g0, [%g1 + CPU_PROFILE_PC]     ! zero kernel PC
1:      ba      pil_interrupt_common            ! must be large-disp branch
          stn   %g0, [%g1 + CPU_PROFILE_UPC]    ! zero user PC
        SET_SIZE(pil14_interrupt)

        ENTRY_NP(tick_rtt)
        !
        ! Load TICK_COMPARE into %o5; if bit 63 is set, then TICK_COMPARE is
        ! disabled.  If TICK_COMPARE is enabled, we know that we need to
        ! reenqueue the interrupt request structure.  We'll then check TICKINT
        ! in SOFTINT; if it's set, then we know that we were in a TICK_COMPARE
        ! interrupt.  In this case, TICK_COMPARE may have been rewritten
        ! recently; we'll compare %o5 to the current time to verify that it's
        ! in the future.
        !
        ! Note that %o5 is live until after 1f.
        ! XXX - there is a subroutine call while %o5 is live!
        !
        RD_TICKCMPR(%o5,%g1,%g2,__LINE__)
        srlx    %o5, TICKINT_DIS_SHFT, %g1
        brnz,pt %g1, 2f
          nop

        rdpr    %pstate, %g5
        andn    %g5, PSTATE_IE, %g1
        wrpr    %g0, %g1, %pstate               ! Disable vec interrupts

        sethi   %hi(cbe_level14_inum), %o1
        ldx     [%o1 + %lo(cbe_level14_inum)], %o1
        call    intr_enqueue_req ! preserves %o5 and %g5
          mov   PIL_14, %o0

        ! Check SOFTINT for TICKINT/STICKINT
        rd      SOFTINT, %o4
        set     (TICK_INT_MASK | STICK_INT_MASK), %o0
        andcc   %o4, %o0, %g0
        bz,a,pn %icc, 2f
          wrpr  %g0, %g5, %pstate               ! Enable vec interrupts

        ! clear TICKINT/STICKINT
        wr      %o0, CLEAR_SOFTINT

        !
        ! Now that we've cleared TICKINT, we can reread %tick and confirm
        ! that the value we programmed is still in the future.  If it isn't,
        ! we need to reprogram TICK_COMPARE to fire as soon as possible.
        !
        GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__)   ! %o0 = tick
        cmp     %o5, %o0                        ! In the future?
        bg,a,pt %xcc, 2f                        ! Yes, drive on.
          wrpr  %g0, %g5, %pstate               !   delay: enable vec intr

        !
        ! If we're here, then we have programmed TICK_COMPARE with a %tick
        ! which is in the past; we'll now load an initial step size, and loop
        ! until we've managed to program TICK_COMPARE to fire in the future.
        !
        mov     8, %o4                          ! 8 = arbitrary inital step
1:      add     %o0, %o4, %o5                   ! Add the step
        WR_TICKCMPR(%o5,%g1,%g2,__LINE__)       ! Write to TICK_CMPR
        GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__)   ! %o0 = tick
        cmp     %o5, %o0                        ! In the future?
        bg,a,pt %xcc, 2f                        ! Yes, drive on.
          wrpr  %g0, %g5, %pstate               !    delay: enable vec intr
        ba      1b                              ! No, try again.
          sllx  %o4, 1, %o4                     !    delay: double step size

2:      ba      current_thread_complete
          nop
        SET_SIZE(tick_rtt)

/*
 * Level-15 interrupt prologue.
 */
       ENTRY_NP(pil15_interrupt)
       CPU_ADDR(%g1, %g2)
       rdpr    %tstate, %g6
       rdpr    %tpc, %g5
       btst    TSTATE_PRIV, %g6                ! trap from supervisor mode?
       bnz,a,pt %xcc, 1f
       stn     %g5, [%g1 + CPU_CPCPROFILE_PC]  ! if so, record kernel PC
       stn     %g5, [%g1 + CPU_CPCPROFILE_UPC] ! if not, record user PC
       ba      pil15_epilogue                  ! must be large-disp branch
       stn     %g0, [%g1 + CPU_CPCPROFILE_PC]  ! zero kernel PC
1:     ba      pil15_epilogue                  ! must be large-disp branch
       stn     %g0, [%g1 + CPU_CPCPROFILE_UPC] ! zero user PC
       SET_SIZE(pil15_interrupt)

/*
 * Prefetch a page_t for write or read, this assumes a linear
 * scan of sequential page_t's.
 */
/* XXXQ These should be inline templates, not functions */
        ENTRY(prefetch_page_w)
        retl
          nop
        SET_SIZE(prefetch_page_w)

        ENTRY(prefetch_page_r)
        retl
          nop
        SET_SIZE(prefetch_page_r)

/*
 * Prefetch struct smap for write.
 */
/* XXXQ These should be inline templates, not functions */
        ENTRY(prefetch_smap_w)
        retl
          nop
        SET_SIZE(prefetch_smap_w)

/*
 * Generic sun4v MMU and Cache operations.
 */

        ENTRY_NP(vtag_flushpage)
        /*
         * flush page from the tlb
         *
         * %o0 = vaddr
         * %o1 = sfmmup
         */
        SFMMU_CPU_CNUM(%o1, %g1, %g2)   /* %g1 = sfmmu cnum on this CPU */

        mov     %g1, %o1
        mov     MAP_ITLB | MAP_DTLB, %o2
        ta      MMU_UNMAP_ADDR
        brz,pt  %o0, 1f
          nop
        ba      panic_bad_hcall
          mov   MMU_UNMAP_ADDR, %o1
1:
        retl
          nop
        SET_SIZE(vtag_flushpage)

        ENTRY_NP(vtag_flushall)
        mov     %g0, %o0        ! XXX no cpu list yet
        mov     %g0, %o1        ! XXX no cpu list yet
        mov     MAP_ITLB | MAP_DTLB, %o2
        mov     MMU_DEMAP_ALL, %o5
        ta      FAST_TRAP
        brz,pt  %o0, 1f
          nop
        ba      panic_bad_hcall
          mov   MMU_DEMAP_ALL, %o1
1:
        retl
          nop
        SET_SIZE(vtag_flushall)

        ENTRY_NP(vtag_unmap_perm_tl1)
        /*
         * x-trap to unmap perm map entry
         * %g1 = vaddr
         * %g2 = ctxnum (KCONTEXT only)
         */
        mov     %o0, %g3
        mov     %o1, %g4
        mov     %o2, %g5
        mov     %o5, %g6
        mov     %g1, %o0
        mov     %g2, %o1
        mov     MAP_ITLB | MAP_DTLB, %o2
        mov     UNMAP_PERM_ADDR, %o5
        ta      FAST_TRAP
        brz,pt  %o0, 1f
        nop

        mov     PTL1_BAD_HCALL, %g1

        cmp     %o0, H_ENOMAP
        move    %xcc, PTL1_BAD_HCALL_UNMAP_PERM_ENOMAP, %g1

        cmp     %o0, H_EINVAL
        move    %xcc, PTL1_BAD_HCALL_UNMAP_PERM_EINVAL, %g1

        ba,a    ptl1_panic
1:
        mov     %g6, %o5
        mov     %g5, %o2
        mov     %g4, %o1
        mov     %g3, %o0
        retry
        SET_SIZE(vtag_unmap_perm_tl1)

        ENTRY_NP(vtag_flushpage_tl1)
        /*
         * x-trap to flush page from tlb and tsb
         *
         * %g1 = vaddr, zero-extended on 32-bit kernel
         * %g2 = sfmmup
         *
         * assumes TSBE_TAG = 0
         */
        srln    %g1, MMU_PAGESHIFT, %g1
        slln    %g1, MMU_PAGESHIFT, %g1                 /* g1 = vaddr */
        mov     %o0, %g3
        mov     %o1, %g4
        mov     %o2, %g5
        mov     %g1, %o0                        /* vaddr */

        SFMMU_CPU_CNUM(%g2, %o1, %g6)   /* %o1 = sfmmu cnum on this CPU */

        mov     MAP_ITLB | MAP_DTLB, %o2
        ta      MMU_UNMAP_ADDR
        brz,pt  %o0, 1f
        nop
          ba    ptl1_panic
        mov     PTL1_BAD_HCALL, %g1
1:
        mov     %g5, %o2
        mov     %g4, %o1
        mov     %g3, %o0
        membar #Sync
        retry
        SET_SIZE(vtag_flushpage_tl1)

        ENTRY_NP(vtag_flush_pgcnt_tl1)
        /*
         * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
         *
         * %g1 = vaddr, zero-extended on 32-bit kernel
         * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits.
         *
         * NOTE: this handler relies on the fact that no
         *      interrupts or traps can occur during the loop
         *      issuing the TLB_DEMAP operations. It is assumed
         *      that interrupts are disabled and this code is
         *      fetching from the kernel locked text address.
         *
         * assumes TSBE_TAG = 0
         */
        srln    %g1, MMU_PAGESHIFT, %g1
        slln    %g1, MMU_PAGESHIFT, %g1         /* g1 = vaddr */
        mov     %o0, %g3
        mov     %o1, %g4
        mov     %o2, %g5

        and     %g2, SFMMU_PGCNT_MASK, %g7      /* g7 = pgcnt - 1 */
        add     %g7, 1, %g7                     /* g7 = pgcnt */

        andn    %g2, SFMMU_PGCNT_MASK, %o0      /* %o0 = sfmmup */

        SFMMU_CPU_CNUM(%o0, %g2, %g6)    /* %g2 = sfmmu cnum on this CPU */

        set     MMU_PAGESIZE, %g6               /* g6 = pgsize */

1:
        mov     %g1, %o0                        /* vaddr */
        mov     %g2, %o1                        /* cnum */
        mov     MAP_ITLB | MAP_DTLB, %o2
        ta      MMU_UNMAP_ADDR
        brz,pt  %o0, 2f
          nop
        ba      ptl1_panic
          mov   PTL1_BAD_HCALL, %g1
2:
        deccc   %g7                             /* decr pgcnt */
        bnz,pt  %icc,1b
          add   %g1, %g6, %g1                   /* go to nextpage */

        mov     %g5, %o2
        mov     %g4, %o1
        mov     %g3, %o0
        membar #Sync
        retry
        SET_SIZE(vtag_flush_pgcnt_tl1)

        ! Not implemented on US1/US2
        ENTRY_NP(vtag_flushall_tl1)
        mov     %o0, %g3
        mov     %o1, %g4
        mov     %o2, %g5
        mov     %o3, %g6        ! XXXQ not used?
        mov     %o5, %g7
        mov     %g0, %o0        ! XXX no cpu list yet
        mov     %g0, %o1        ! XXX no cpu list yet
        mov     MAP_ITLB | MAP_DTLB, %o2
        mov     MMU_DEMAP_ALL, %o5
        ta      FAST_TRAP
        brz,pt  %o0, 1f
          nop
        ba      ptl1_panic
          mov   PTL1_BAD_HCALL, %g1
1:
        mov     %g7, %o5
        mov     %g6, %o3        ! XXXQ not used?
        mov     %g5, %o2
        mov     %g4, %o1
        mov     %g3, %o0
        retry
        SET_SIZE(vtag_flushall_tl1)

/*
 * flush_instr_mem:
 *      Flush a portion of the I-$ starting at vaddr
 *      %o0 vaddr
 *      %o1 bytes to be flushed
 */

        ENTRY(flush_instr_mem)
        membar  #StoreStore                             ! Ensure the stores
                                                        ! are globally visible
1:
        flush   %o0
        subcc   %o1, ICACHE_FLUSHSZ, %o1                ! bytes = bytes-0x20
        bgu,pt  %ncc, 1b
          add   %o0, ICACHE_FLUSHSZ, %o0                ! vaddr = vaddr+0x20

        retl
          nop
        SET_SIZE(flush_instr_mem)

#if !defined(CUSTOM_FPZERO)

/*
 * fp_zero() - clear all fp data registers and the fsr
 */

.global fp_zero_zero
.align 8
fp_zero_zero:
        .xword  0

        ENTRY_NP(fp_zero)
        sethi   %hi(fp_zero_zero), %o0
        ldx     [%o0 + %lo(fp_zero_zero)], %fsr
        ldd     [%o0 + %lo(fp_zero_zero)], %f0
        fmovd   %f0, %f2
        fmovd   %f0, %f4
        fmovd   %f0, %f6
        fmovd   %f0, %f8
        fmovd   %f0, %f10
        fmovd   %f0, %f12
        fmovd   %f0, %f14
        fmovd   %f0, %f16
        fmovd   %f0, %f18
        fmovd   %f0, %f20
        fmovd   %f0, %f22
        fmovd   %f0, %f24
        fmovd   %f0, %f26
        fmovd   %f0, %f28
        fmovd   %f0, %f30
        fmovd   %f0, %f32
        fmovd   %f0, %f34
        fmovd   %f0, %f36
        fmovd   %f0, %f38
        fmovd   %f0, %f40
        fmovd   %f0, %f42
        fmovd   %f0, %f44
        fmovd   %f0, %f46
        fmovd   %f0, %f48
        fmovd   %f0, %f50
        fmovd   %f0, %f52
        fmovd   %f0, %f54
        fmovd   %f0, %f56
        fmovd   %f0, %f58
        fmovd   %f0, %f60
        retl
        fmovd   %f0, %f62
        SET_SIZE(fp_zero)

#endif  /* CUSTOM_FPZERO */