/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Kernel Physical Mapping (segkpm) hat interface routines for sun4u.
 */

#include <sys/types.h>
#include <vm/hat.h>
#include <vm/hat_sfmmu.h>
#include <vm/page.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/machsystm.h>
#include <vm/seg_kpm.h>
#include <sys/cpu_module.h>
#include <vm/mach_kpm.h>

/* kpm prototypes */
static caddr_t  sfmmu_kpm_mapin(page_t *);
static void     sfmmu_kpm_mapout(page_t *, caddr_t);
static int      sfmmu_kpme_lookup(struct kpme *, page_t *);
static void     sfmmu_kpme_add(struct kpme *, page_t *);
static void     sfmmu_kpme_sub(struct kpme *, page_t *);
static caddr_t  sfmmu_kpm_getvaddr(page_t *, int *);
static int      sfmmu_kpm_fault(caddr_t, struct memseg *, page_t *);
static int      sfmmu_kpm_fault_small(caddr_t, struct memseg *, page_t *);
static void     sfmmu_kpm_vac_conflict(page_t *, caddr_t);
void    sfmmu_kpm_pageunload(page_t *);
void    sfmmu_kpm_vac_unload(page_t *, caddr_t);
static void     sfmmu_kpm_demap_large(caddr_t);
static void     sfmmu_kpm_demap_small(caddr_t);
static void     sfmmu_kpm_demap_tlbs(caddr_t);
void    sfmmu_kpm_hme_unload(page_t *);
kpm_hlk_t *sfmmu_kpm_kpmp_enter(page_t *, pgcnt_t);
void    sfmmu_kpm_kpmp_exit(kpm_hlk_t *kpmp);
void    sfmmu_kpm_page_cache(page_t *, int, int);

extern uint_t vac_colors;

/*
 * Kernel Physical Mapping (kpm) facility
 */

void
mach_kpm_init()
{}

/* -- hat_kpm interface section -- */

/*
 * Mapin a locked page and return the vaddr.
 * When a kpme is provided by the caller it is added to
 * the page p_kpmelist. The page to be mapped in must
 * be at least read locked (p_selock).
 */
caddr_t
hat_kpm_mapin(struct page *pp, struct kpme *kpme)
{
        kmutex_t        *pml;
        caddr_t         vaddr;

        if (kpm_enable == 0) {
                cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set");
                return ((caddr_t)NULL);
        }

        if (pp == NULL || PAGE_LOCKED(pp) == 0) {
                cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked");
                return ((caddr_t)NULL);
        }

        pml = sfmmu_mlist_enter(pp);
        ASSERT(pp->p_kpmref >= 0);

        vaddr = (pp->p_kpmref == 0) ?
            sfmmu_kpm_mapin(pp) : hat_kpm_page2va(pp, 1);

        if (kpme != NULL) {
                /*
                 * Tolerate multiple mapins for the same kpme to avoid
                 * the need for an extra serialization.
                 */
                if ((sfmmu_kpme_lookup(kpme, pp)) == 0)
                        sfmmu_kpme_add(kpme, pp);

                ASSERT(pp->p_kpmref > 0);

        } else {
                pp->p_kpmref++;
        }

        sfmmu_mlist_exit(pml);
        return (vaddr);
}

/*
 * Mapout a locked page.
 * When a kpme is provided by the caller it is removed from
 * the page p_kpmelist. The page to be mapped out must be at
 * least read locked (p_selock).
 * Note: The seg_kpm layer provides a mapout interface for the
 * case that a kpme is used and the underlying page is unlocked.
 * This can be used instead of calling this function directly.
 */
void
hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr)
{
        kmutex_t        *pml;

        if (kpm_enable == 0) {
                cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set");
                return;
        }

        if (IS_KPM_ADDR(vaddr) == 0) {
                cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address");
                return;
        }

        if (pp == NULL || PAGE_LOCKED(pp) == 0) {
                cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked");
                return;
        }

        if (kpme != NULL) {
                ASSERT(pp == kpme->kpe_page);
                pp = kpme->kpe_page;
                pml = sfmmu_mlist_enter(pp);

                if (sfmmu_kpme_lookup(kpme, pp) == 0)
                        panic("hat_kpm_mapout: kpme not found pp=%p",
                            (void *)pp);

                ASSERT(pp->p_kpmref > 0);
                sfmmu_kpme_sub(kpme, pp);

        } else {
                pml = sfmmu_mlist_enter(pp);
                pp->p_kpmref--;
        }

        ASSERT(pp->p_kpmref >= 0);
        if (pp->p_kpmref == 0)
                sfmmu_kpm_mapout(pp, vaddr);

        sfmmu_mlist_exit(pml);
}

/*
 * hat_kpm_mapin_pfn is used to obtain a kpm mapping for physical
 * memory addresses that are not described by a page_t.  It can
 * only be supported if vac_colors=1, because there is no page_t
 * and corresponding kpm_page_t to track VAC conflicts.  Currently,
 * this may not be used on pfn's backed by page_t's, because the
 * kpm state may not be consistent in hat_kpm_fault if the page is
 * mapped using both this routine and hat_kpm_mapin.  KPM should be
 * cleaned up on sun4u/vac_colors=1 to be minimal as on sun4v.
 * The caller must only pass pfn's for valid physical addresses; violation
 * of this rule will cause panic.
 */
caddr_t
hat_kpm_mapin_pfn(pfn_t pfn)
{
        caddr_t paddr, vaddr;
        tte_t tte;
        uint_t szc = kpm_smallpages ? TTE8K : TTE4M;
        uint_t shift = kpm_smallpages ? MMU_PAGESHIFT : MMU_PAGESHIFT4M;

        if (kpm_enable == 0 || vac_colors > 1 ||
            page_numtomemseg_nolock(pfn) != NULL)
                return ((caddr_t)NULL);

        paddr = (caddr_t)ptob(pfn);
        vaddr = (uintptr_t)kpm_vbase + paddr;

        KPM_TTE_VCACHED(tte.ll, pfn, szc);
        sfmmu_kpm_load_tsb(vaddr, &tte, shift);

        return (vaddr);
}

/*ARGSUSED*/
void
hat_kpm_mapout_pfn(pfn_t pfn)
{
        /* empty */
}

/*
 * Return the kpm virtual address for the page at pp.
 * If checkswap is non zero and the page is backed by a
 * swap vnode the physical address is used rather than
 * p_offset to determine the kpm region.
 * Note: The function has to be used w/ extreme care. The
 * stability of the page identity is in the responsibility
 * of the caller.
 */
/*ARGSUSED*/
caddr_t
hat_kpm_page2va(struct page *pp, int checkswap)
{
        int             vcolor, vcolor_pa;
        uintptr_t       paddr, vaddr;

        ASSERT(kpm_enable);

        paddr = ptob(pp->p_pagenum);
        vcolor_pa = addr_to_vcolor(paddr);

        if (checkswap && pp->p_vnode && IS_SWAPFSVP(pp->p_vnode))
                vcolor = (PP_ISNC(pp)) ? vcolor_pa : PP_GET_VCOLOR(pp);
        else
                vcolor = addr_to_vcolor(pp->p_offset);

        vaddr = (uintptr_t)kpm_vbase + paddr;

        if (vcolor_pa != vcolor) {
                vaddr += ((uintptr_t)(vcolor - vcolor_pa) << MMU_PAGESHIFT);
                vaddr += (vcolor_pa > vcolor) ?
                    ((uintptr_t)vcolor_pa << kpm_size_shift) :
                    ((uintptr_t)(vcolor - vcolor_pa) << kpm_size_shift);
        }

        return ((caddr_t)vaddr);
}

/*
 * Return the page for the kpm virtual address vaddr.
 * Caller is responsible for the kpm mapping and lock
 * state of the page.
 */
page_t *
hat_kpm_vaddr2page(caddr_t vaddr)
{
        uintptr_t       paddr;
        pfn_t           pfn;

        ASSERT(IS_KPM_ADDR(vaddr));

        SFMMU_KPM_VTOP(vaddr, paddr);
        pfn = (pfn_t)btop(paddr);

        return (page_numtopp_nolock(pfn));
}

/* page to kpm_page */
#define PP2KPMPG(pp, kp) {                                              \
        struct memseg   *mseg;                                          \
        pgcnt_t         inx;                                            \
        pfn_t           pfn;                                            \
                                                                        \
        pfn = pp->p_pagenum;                                            \
        mseg = page_numtomemseg_nolock(pfn);                            \
        ASSERT(mseg);                                                   \
        inx = ptokpmp(kpmptop(ptokpmp(pfn)) - mseg->kpm_pbase);         \
        ASSERT(inx < mseg->kpm_nkpmpgs);                                \
        kp = &mseg->kpm_pages[inx];                                     \
}

/* page to kpm_spage */
#define PP2KPMSPG(pp, ksp) {                                            \
        struct memseg   *mseg;                                          \
        pgcnt_t         inx;                                            \
        pfn_t           pfn;                                            \
                                                                        \
        pfn = pp->p_pagenum;                                            \
        mseg = page_numtomemseg_nolock(pfn);                            \
        ASSERT(mseg);                                                   \
        inx = pfn - mseg->kpm_pbase;                                    \
        ksp = &mseg->kpm_spages[inx];                                   \
}

/*
 * hat_kpm_fault is called from segkpm_fault when a kpm tsbmiss occurred
 * which could not be resolved by the trap level tsbmiss handler for the
 * following reasons:
 * . The vaddr is in VAC alias range (always PAGESIZE mapping size).
 * . The kpm (s)page range of vaddr is in a VAC alias prevention state.
 * . tsbmiss handling at trap level is not desired (DEBUG kernel only,
 *   kpm_tsbmtl == 0).
 */
int
hat_kpm_fault(struct hat *hat, caddr_t vaddr)
{
        int             error;
        uintptr_t       paddr;
        pfn_t           pfn;
        struct memseg   *mseg;
        page_t  *pp;

        if (kpm_enable == 0) {
                cmn_err(CE_WARN, "hat_kpm_fault: kpm_enable not set");
                return (ENOTSUP);
        }

        ASSERT(hat == ksfmmup);
        ASSERT(IS_KPM_ADDR(vaddr));

        SFMMU_KPM_VTOP(vaddr, paddr);
        pfn = (pfn_t)btop(paddr);
        if ((mseg = page_numtomemseg_nolock(pfn)) != NULL) {
                pp = &mseg->pages[(pgcnt_t)(pfn - mseg->pages_base)];
                ASSERT((pfn_t)pp->p_pagenum == pfn);
        }

        /*
         * hat_kpm_mapin_pfn may add a kpm translation for memory that falls
         * outside of memsegs.  Check for this case and provide the translation
         * here.
         */
        if (vac_colors == 1 && mseg == NULL) {
                tte_t tte;
                uint_t szc = kpm_smallpages ? TTE8K : TTE4M;
                uint_t shift = kpm_smallpages ? MMU_PAGESHIFT : MMU_PAGESHIFT4M;

                ASSERT(address_in_memlist(phys_install, paddr, 1));
                KPM_TTE_VCACHED(tte.ll, pfn, szc);
                sfmmu_kpm_load_tsb(vaddr, &tte, shift);
                error = 0;
        } else if (mseg == NULL || !PAGE_LOCKED(pp))
                error = EFAULT;
        else if (kpm_smallpages == 0)
                error = sfmmu_kpm_fault(vaddr, mseg, pp);
        else
                error = sfmmu_kpm_fault_small(vaddr, mseg, pp);

        return (error);
}

/*
 * memseg_hash[] was cleared, need to clear memseg_phash[] too.
 */
void
hat_kpm_mseghash_clear(int nentries)
{
        pgcnt_t i;

        if (kpm_enable == 0)
                return;

        for (i = 0; i < nentries; i++)
                memseg_phash[i] = MSEG_NULLPTR_PA;
}

/*
 * Update memseg_phash[inx] when memseg_hash[inx] was changed.
 */
void
hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp)
{
        if (kpm_enable == 0)
                return;

        memseg_phash[inx] = (msp) ? va_to_pa(msp) : MSEG_NULLPTR_PA;
}

/*
 * Update kpm memseg members from basic memseg info.
 */
void
hat_kpm_addmem_mseg_update(struct memseg *msp, pgcnt_t nkpmpgs,
        offset_t kpm_pages_off)
{
        if (kpm_enable == 0)
                return;

        msp->kpm_pages = (kpm_page_t *)((caddr_t)msp->pages + kpm_pages_off);
        msp->kpm_nkpmpgs = nkpmpgs;
        msp->kpm_pbase = kpmptop(ptokpmp(msp->pages_base));
        msp->pagespa = va_to_pa(msp->pages);
        msp->epagespa = va_to_pa(msp->epages);
        msp->kpm_pagespa = va_to_pa(msp->kpm_pages);
}

/*
 * Setup nextpa when a memseg is inserted.
 * Assumes that the memsegslock is already held.
 */
void
hat_kpm_addmem_mseg_insert(struct memseg *msp)
{
        if (kpm_enable == 0)
                return;

        ASSERT(memsegs_lock_held());
        msp->nextpa = (memsegs) ? va_to_pa(memsegs) : MSEG_NULLPTR_PA;
}

/*
 * Setup memsegspa when a memseg is (head) inserted.
 * Called before memsegs is updated to complete a
 * memseg insert operation.
 * Assumes that the memsegslock is already held.
 */
void
hat_kpm_addmem_memsegs_update(struct memseg *msp)
{
        if (kpm_enable == 0)
                return;

        ASSERT(memsegs_lock_held());
        ASSERT(memsegs);
        memsegspa = va_to_pa(msp);
}

/*
 * Return end of metadata for an already setup memseg.
 *
 * Note: kpm_pages and kpm_spages are aliases and the underlying
 * member of struct memseg is a union, therefore they always have
 * the same address within a memseg. They must be differentiated
 * when pointer arithmetic is used with them.
 */
caddr_t
hat_kpm_mseg_reuse(struct memseg *msp)
{
        caddr_t end;

        if (kpm_smallpages == 0)
                end = (caddr_t)(msp->kpm_pages + msp->kpm_nkpmpgs);
        else
                end = (caddr_t)(msp->kpm_spages + msp->kpm_nkpmpgs);

        return (end);
}

/*
 * Update memsegspa (when first memseg in list
 * is deleted) or nextpa  when a memseg deleted.
 * Assumes that the memsegslock is already held.
 */
void
hat_kpm_delmem_mseg_update(struct memseg *msp, struct memseg **mspp)
{
        struct memseg *lmsp;

        if (kpm_enable == 0)
                return;

        ASSERT(memsegs_lock_held());

        if (mspp == &memsegs) {
                memsegspa = (msp->next) ?
                    va_to_pa(msp->next) : MSEG_NULLPTR_PA;
        } else {
                lmsp = (struct memseg *)
                    ((uint64_t)mspp - offsetof(struct memseg, next));
                lmsp->nextpa = (msp->next) ?
                    va_to_pa(msp->next) : MSEG_NULLPTR_PA;
        }
}

/*
 * Update kpm members for all memseg's involved in a split operation
 * and do the atomic update of the physical memseg chain.
 *
 * Note: kpm_pages and kpm_spages are aliases and the underlying member
 * of struct memseg is a union, therefore they always have the same
 * address within a memseg. With that the direct assignments and
 * va_to_pa conversions below don't have to be distinguished wrt. to
 * kpm_smallpages. They must be differentiated when pointer arithmetic
 * is used with them.
 *
 * Assumes that the memsegslock is already held.
 */
void
hat_kpm_split_mseg_update(struct memseg *msp, struct memseg **mspp,
        struct memseg *lo, struct memseg *mid, struct memseg *hi)
{
        pgcnt_t start, end, kbase, kstart, num;
        struct memseg *lmsp;

        if (kpm_enable == 0)
                return;

        ASSERT(memsegs_lock_held());
        ASSERT(msp && mid && msp->kpm_pages);

        kbase = ptokpmp(msp->kpm_pbase);

        if (lo) {
                num = lo->pages_end - lo->pages_base;
                start = kpmptop(ptokpmp(lo->pages_base));
                /* align end to kpm page size granularity */
                end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs;
                lo->kpm_pbase = start;
                lo->kpm_nkpmpgs = ptokpmp(end - start);
                lo->kpm_pages = msp->kpm_pages;
                lo->kpm_pagespa = va_to_pa(lo->kpm_pages);
                lo->pagespa = va_to_pa(lo->pages);
                lo->epagespa = va_to_pa(lo->epages);
                lo->nextpa = va_to_pa(lo->next);
        }

        /* mid */
        num = mid->pages_end - mid->pages_base;
        kstart = ptokpmp(mid->pages_base);
        start = kpmptop(kstart);
        /* align end to kpm page size granularity */
        end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs;
        mid->kpm_pbase = start;
        mid->kpm_nkpmpgs = ptokpmp(end - start);
        if (kpm_smallpages == 0) {
                mid->kpm_pages = msp->kpm_pages + (kstart - kbase);
        } else {
                mid->kpm_spages = msp->kpm_spages + (kstart - kbase);
        }
        mid->kpm_pagespa = va_to_pa(mid->kpm_pages);
        mid->pagespa = va_to_pa(mid->pages);
        mid->epagespa = va_to_pa(mid->epages);
        mid->nextpa = (mid->next) ?  va_to_pa(mid->next) : MSEG_NULLPTR_PA;

        if (hi) {
                num = hi->pages_end - hi->pages_base;
                kstart = ptokpmp(hi->pages_base);
                start = kpmptop(kstart);
                /* align end to kpm page size granularity */
                end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs;
                hi->kpm_pbase = start;
                hi->kpm_nkpmpgs = ptokpmp(end - start);
                if (kpm_smallpages == 0) {
                        hi->kpm_pages = msp->kpm_pages + (kstart - kbase);
                } else {
                        hi->kpm_spages = msp->kpm_spages + (kstart - kbase);
                }
                hi->kpm_pagespa = va_to_pa(hi->kpm_pages);
                hi->pagespa = va_to_pa(hi->pages);
                hi->epagespa = va_to_pa(hi->epages);
                hi->nextpa = (hi->next) ? va_to_pa(hi->next) : MSEG_NULLPTR_PA;
        }

        /*
         * Atomic update of the physical memseg chain
         */
        if (mspp == &memsegs) {
                memsegspa = (lo) ? va_to_pa(lo) : va_to_pa(mid);
        } else {
                lmsp = (struct memseg *)
                    ((uint64_t)mspp - offsetof(struct memseg, next));
                lmsp->nextpa = (lo) ? va_to_pa(lo) : va_to_pa(mid);
        }
}

/*
 * Walk the memsegs chain, applying func to each memseg span and vcolor.
 */
void
hat_kpm_walk(void (*func)(void *, void *, size_t), void *arg)
{
        pfn_t   pbase, pend;
        int     vcolor;
        void    *base;
        size_t  size;
        struct memseg *msp;

        for (msp = memsegs; msp; msp = msp->next) {
                pbase = msp->pages_base;
                pend = msp->pages_end;
                for (vcolor = 0; vcolor < vac_colors; vcolor++) {
                        base = ptob(pbase) + kpm_vbase + kpm_size * vcolor;
                        size = ptob(pend - pbase);
                        func(arg, base, size);
                }
        }
}


/* -- sfmmu_kpm internal section -- */

/*
 * Return the page frame number if a valid segkpm mapping exists
 * for vaddr, otherwise return PFN_INVALID. No locks are grabbed.
 * Should only be used by other sfmmu routines.
 */
pfn_t
sfmmu_kpm_vatopfn(caddr_t vaddr)
{
        uintptr_t       paddr;
        pfn_t           pfn;
        page_t  *pp;

        ASSERT(kpm_enable && IS_KPM_ADDR(vaddr));

        SFMMU_KPM_VTOP(vaddr, paddr);
        pfn = (pfn_t)btop(paddr);
        pp = page_numtopp_nolock(pfn);
        if (pp && pp->p_kpmref)
                return (pfn);
        else
                return ((pfn_t)PFN_INVALID);
}

/*
 * Lookup a kpme in the p_kpmelist.
 */
static int
sfmmu_kpme_lookup(struct kpme *kpme, page_t *pp)
{
        struct kpme     *p;

        for (p = pp->p_kpmelist; p; p = p->kpe_next) {
                if (p == kpme)
                        return (1);
        }
        return (0);
}

/*
 * Insert a kpme into the p_kpmelist and increment
 * the per page kpm reference count.
 */
static void
sfmmu_kpme_add(struct kpme *kpme, page_t *pp)
{
        ASSERT(pp->p_kpmref >= 0);

        /* head insert */
        kpme->kpe_prev = NULL;
        kpme->kpe_next = pp->p_kpmelist;

        if (pp->p_kpmelist)
                pp->p_kpmelist->kpe_prev = kpme;

        pp->p_kpmelist = kpme;
        kpme->kpe_page = pp;
        pp->p_kpmref++;
}

/*
 * Remove a kpme from the p_kpmelist and decrement
 * the per page kpm reference count.
 */
static void
sfmmu_kpme_sub(struct kpme *kpme, page_t *pp)
{
        ASSERT(pp->p_kpmref > 0);

        if (kpme->kpe_prev) {
                ASSERT(pp->p_kpmelist != kpme);
                ASSERT(kpme->kpe_prev->kpe_page == pp);
                kpme->kpe_prev->kpe_next = kpme->kpe_next;
        } else {
                ASSERT(pp->p_kpmelist == kpme);
                pp->p_kpmelist = kpme->kpe_next;
        }

        if (kpme->kpe_next) {
                ASSERT(kpme->kpe_next->kpe_page == pp);
                kpme->kpe_next->kpe_prev = kpme->kpe_prev;
        }

        kpme->kpe_next = kpme->kpe_prev = NULL;
        kpme->kpe_page = NULL;
        pp->p_kpmref--;
}

/*
 * Mapin a single page, it is called every time a page changes it's state
 * from kpm-unmapped to kpm-mapped. It may not be called, when only a new
 * kpm instance does a mapin and wants to share the mapping.
 * Assumes that the mlist mutex is already grabbed.
 */
static caddr_t
sfmmu_kpm_mapin(page_t *pp)
{
        kpm_page_t      *kp;
        kpm_hlk_t       *kpmp;
        caddr_t         vaddr;
        int             kpm_vac_range;
        pfn_t           pfn;
        tte_t           tte;
        kmutex_t        *pmtx;
        int             uncached;
        kpm_spage_t     *ksp;
        kpm_shlk_t      *kpmsp;
        int             oldval;

        ASSERT(sfmmu_mlist_held(pp));
        ASSERT(pp->p_kpmref == 0);

        vaddr = sfmmu_kpm_getvaddr(pp, &kpm_vac_range);

        ASSERT(IS_KPM_ADDR(vaddr));
        uncached = PP_ISNC(pp);
        pfn = pp->p_pagenum;

        if (kpm_smallpages)
                goto smallpages_mapin;

        PP2KPMPG(pp, kp);

        kpmp = KPMP_HASH(kp);
        mutex_enter(&kpmp->khl_mutex);

        ASSERT(PP_ISKPMC(pp) == 0);
        ASSERT(PP_ISKPMS(pp) == 0);

        if (uncached) {
                /* ASSERT(pp->p_share); XXX use hat_page_getshare */
                if (kpm_vac_range == 0) {
                        if (kp->kp_refcnts == 0) {
                                /*
                                 * Must remove large page mapping if it exists.
                                 * Pages in uncached state can only be mapped
                                 * small (PAGESIZE) within the regular kpm
                                 * range.
                                 */
                                if (kp->kp_refcntc == -1) {
                                        /* remove go indication */
                                        sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
                                            &kpmp->khl_lock, KPMTSBM_STOP);
                                }
                                if (kp->kp_refcnt > 0 && kp->kp_refcntc == 0)
                                        sfmmu_kpm_demap_large(vaddr);
                        }
                        ASSERT(kp->kp_refcntc >= 0);
                        kp->kp_refcntc++;
                }
                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMC(pp);
                sfmmu_page_exit(pmtx);
        }

        if ((kp->kp_refcntc > 0 || kp->kp_refcnts > 0) && kpm_vac_range == 0) {
                /*
                 * Have to do a small (PAGESIZE) mapin within this kpm_page
                 * range since it is marked to be in VAC conflict mode or
                 * when there are still other small mappings around.
                 */

                /* tte assembly */
                if (uncached == 0)
                        KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
                else
                        KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);

                /* tsb dropin */
                sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);

                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMS(pp);
                sfmmu_page_exit(pmtx);

                kp->kp_refcnts++;
                ASSERT(kp->kp_refcnts > 0);
                goto exit;
        }

        if (kpm_vac_range == 0) {
                /*
                 * Fast path / regular case, no VAC conflict handling
                 * in progress within this kpm_page range.
                 */
                if (kp->kp_refcnt == 0) {

                        /* tte assembly */
                        KPM_TTE_VCACHED(tte.ll, pfn, TTE4M);

                        /* tsb dropin */
                        sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT4M);

                        /* Set go flag for TL tsbmiss handler */
                        if (kp->kp_refcntc == 0)
                                sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
                                    &kpmp->khl_lock, KPMTSBM_START);

                        ASSERT(kp->kp_refcntc == -1);
                }
                kp->kp_refcnt++;
                ASSERT(kp->kp_refcnt);

        } else {
                /*
                 * The page is not setup according to the common VAC
                 * prevention rules for the regular and kpm mapping layer
                 * E.g. the page layer was not able to deliver a right
                 * vcolor'ed page for a given vaddr corresponding to
                 * the wanted p_offset. It has to be mapped in small in
                 * within the corresponding kpm vac range in order to
                 * prevent VAC alias conflicts.
                 */

                /* tte assembly */
                if (uncached == 0) {
                        KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
                } else {
                        KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);
                }

                /* tsb dropin */
                sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);

                kp->kp_refcnta++;
                if (kp->kp_refcntc == -1) {
                        ASSERT(kp->kp_refcnt > 0);

                        /* remove go indication */
                        sfmmu_kpm_tsbmtl(&kp->kp_refcntc, &kpmp->khl_lock,
                            KPMTSBM_STOP);
                }
                ASSERT(kp->kp_refcntc >= 0);
        }
exit:
        mutex_exit(&kpmp->khl_mutex);
        return (vaddr);

smallpages_mapin:
        if (uncached == 0) {
                /* tte assembly */
                KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
        } else {
                /*
                 * Just in case this same page was mapped cacheable prior to
                 * this and the old tte remains in tlb.
                 */
                sfmmu_kpm_demap_small(vaddr);

                /* ASSERT(pp->p_share); XXX use hat_page_getshare */
                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMC(pp);
                sfmmu_page_exit(pmtx);
                /* tte assembly */
                KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);
        }

        /* tsb dropin */
        sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);

        PP2KPMSPG(pp, ksp);
        kpmsp = KPMP_SHASH(ksp);

        oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag, &kpmsp->kshl_lock,
            (uncached) ? (KPM_MAPPED_GO | KPM_MAPPEDSC) :
            (KPM_MAPPED_GO | KPM_MAPPEDS));

        if (oldval != 0)
                panic("sfmmu_kpm_mapin: stale smallpages mapping");

        return (vaddr);
}

/*
 * Mapout a single page, it is called every time a page changes it's state
 * from kpm-mapped to kpm-unmapped. It may not be called, when only a kpm
 * instance calls mapout and there are still other instances mapping the
 * page. Assumes that the mlist mutex is already grabbed.
 *
 * Note: In normal mode (no VAC conflict prevention pending) TLB's are
 * not flushed. This is the core segkpm behavior to avoid xcalls. It is
 * no problem because a translation from a segkpm virtual address to a
 * physical address is always the same. The only downside is a slighty
 * increased window of vulnerability for misbehaving _kernel_ modules.
 */
static void
sfmmu_kpm_mapout(page_t *pp, caddr_t vaddr)
{
        kpm_page_t      *kp;
        kpm_hlk_t       *kpmp;
        int             alias_range;
        kmutex_t        *pmtx;
        kpm_spage_t     *ksp;
        kpm_shlk_t      *kpmsp;
        int             oldval;

        ASSERT(sfmmu_mlist_held(pp));
        ASSERT(pp->p_kpmref == 0);

        alias_range = IS_KPM_ALIAS_RANGE(vaddr);

        if (kpm_smallpages)
                goto smallpages_mapout;

        PP2KPMPG(pp, kp);
        kpmp = KPMP_HASH(kp);
        mutex_enter(&kpmp->khl_mutex);

        if (alias_range) {
                ASSERT(PP_ISKPMS(pp) == 0);
                if (kp->kp_refcnta <= 0) {
                        panic("sfmmu_kpm_mapout: bad refcnta kp=%p",
                            (void *)kp);
                }

                if (PP_ISTNC(pp))  {
                        if (PP_ISKPMC(pp) == 0) {
                                /*
                                 * Uncached kpm mappings must always have
                                 * forced "small page" mode.
                                 */
                                panic("sfmmu_kpm_mapout: uncached page not "
                                    "kpm marked");
                        }
                        sfmmu_kpm_demap_small(vaddr);

                        pmtx = sfmmu_page_enter(pp);
                        PP_CLRKPMC(pp);
                        sfmmu_page_exit(pmtx);

                        /*
                         * Check if we can resume cached mode. This might
                         * be the case if the kpm mapping was the only
                         * mapping in conflict with other non rule
                         * compliant mappings. The page is no more marked
                         * as kpm mapped, so the conv_tnc path will not
                         * change kpm state.
                         */
                        conv_tnc(pp, TTE8K);

                } else if (PP_ISKPMC(pp) == 0) {
                        /* remove TSB entry only */
                        sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT);

                } else {
                        /* already demapped */
                        pmtx = sfmmu_page_enter(pp);
                        PP_CLRKPMC(pp);
                        sfmmu_page_exit(pmtx);
                }
                kp->kp_refcnta--;
                goto exit;
        }

        if (kp->kp_refcntc <= 0 && kp->kp_refcnts == 0) {
                /*
                 * Fast path / regular case.
                 */
                ASSERT(kp->kp_refcntc >= -1);
                ASSERT(!(pp->p_nrm & (P_KPMC | P_KPMS | P_TNC | P_PNC)));

                if (kp->kp_refcnt <= 0)
                        panic("sfmmu_kpm_mapout: bad refcnt kp=%p", (void *)kp);

                if (--kp->kp_refcnt == 0) {
                        /* remove go indication */
                        if (kp->kp_refcntc == -1) {
                                sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
                                    &kpmp->khl_lock, KPMTSBM_STOP);
                        }
                        ASSERT(kp->kp_refcntc == 0);

                        /* remove TSB entry */
                        sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M);
#ifdef  DEBUG
                        if (kpm_tlb_flush)
                                sfmmu_kpm_demap_tlbs(vaddr);
#endif
                }

        } else {
                /*
                 * The VAC alias path.
                 * We come here if the kpm vaddr is not in any alias_range
                 * and we are unmapping a page within the regular kpm_page
                 * range. The kpm_page either holds conflict pages and/or
                 * is in "small page" mode. If the page is not marked
                 * P_KPMS it couldn't have a valid PAGESIZE sized TSB
                 * entry. Dcache flushing is done lazy and follows the
                 * rules of the regular virtual page coloring scheme.
                 *
                 * Per page states and required actions:
                 *   P_KPMC: remove a kpm mapping that is conflicting.
                 *   P_KPMS: remove a small kpm mapping within a kpm_page.
                 *   P_TNC:  check if we can re-cache the page.
                 *   P_PNC:  we cannot re-cache, sorry.
                 * Per kpm_page:
                 *   kp_refcntc > 0: page is part of a kpm_page with conflicts.
                 *   kp_refcnts > 0: rm a small mapped page within a kpm_page.
                 */

                if (PP_ISKPMS(pp)) {
                        if (kp->kp_refcnts < 1) {
                                panic("sfmmu_kpm_mapout: bad refcnts kp=%p",
                                    (void *)kp);
                        }
                        sfmmu_kpm_demap_small(vaddr);

                        /*
                         * Check if we can resume cached mode. This might
                         * be the case if the kpm mapping was the only
                         * mapping in conflict with other non rule
                         * compliant mappings. The page is no more marked
                         * as kpm mapped, so the conv_tnc path will not
                         * change kpm state.
                         */
                        if (PP_ISTNC(pp))  {
                                if (!PP_ISKPMC(pp)) {
                                        /*
                                         * Uncached kpm mappings must always
                                         * have forced "small page" mode.
                                         */
                                        panic("sfmmu_kpm_mapout: uncached "
                                            "page not kpm marked");
                                }
                                conv_tnc(pp, TTE8K);
                        }
                        kp->kp_refcnts--;
                        kp->kp_refcnt++;
                        pmtx = sfmmu_page_enter(pp);
                        PP_CLRKPMS(pp);
                        sfmmu_page_exit(pmtx);
                }

                if (PP_ISKPMC(pp)) {
                        if (kp->kp_refcntc < 1) {
                                panic("sfmmu_kpm_mapout: bad refcntc kp=%p",
                                    (void *)kp);
                        }
                        pmtx = sfmmu_page_enter(pp);
                        PP_CLRKPMC(pp);
                        sfmmu_page_exit(pmtx);
                        kp->kp_refcntc--;
                }

                if (kp->kp_refcnt-- < 1)
                        panic("sfmmu_kpm_mapout: bad refcnt kp=%p", (void *)kp);
        }
exit:
        mutex_exit(&kpmp->khl_mutex);
        return;

smallpages_mapout:
        PP2KPMSPG(pp, ksp);
        kpmsp = KPMP_SHASH(ksp);

        if (PP_ISKPMC(pp) == 0) {
                oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
                    &kpmsp->kshl_lock, 0);

                if (oldval != KPM_MAPPEDS) {
                        /*
                         * When we're called after sfmmu_kpm_hme_unload,
                         * KPM_MAPPEDSC is valid too.
                         */
                        if (oldval != KPM_MAPPEDSC)
                                panic("sfmmu_kpm_mapout: incorrect mapping");
                }

                /* remove TSB entry */
                sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT);
#ifdef  DEBUG
                if (kpm_tlb_flush)
                        sfmmu_kpm_demap_tlbs(vaddr);
#endif

        } else if (PP_ISTNC(pp)) {
                oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
                    &kpmsp->kshl_lock, 0);

                if (oldval != KPM_MAPPEDSC || PP_ISKPMC(pp) == 0)
                        panic("sfmmu_kpm_mapout: inconsistent TNC mapping");

                sfmmu_kpm_demap_small(vaddr);

                pmtx = sfmmu_page_enter(pp);
                PP_CLRKPMC(pp);
                sfmmu_page_exit(pmtx);

                /*
                 * Check if we can resume cached mode. This might be
                 * the case if the kpm mapping was the only mapping
                 * in conflict with other non rule compliant mappings.
                 * The page is no more marked as kpm mapped, so the
                 * conv_tnc path will not change the kpm state.
                 */
                conv_tnc(pp, TTE8K);

        } else {
                oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
                    &kpmsp->kshl_lock, 0);

                if (oldval != KPM_MAPPEDSC)
                        panic("sfmmu_kpm_mapout: inconsistent mapping");

                pmtx = sfmmu_page_enter(pp);
                PP_CLRKPMC(pp);
                sfmmu_page_exit(pmtx);
        }
}

#define abs(x)  ((x) < 0 ? -(x) : (x))

/*
 * Determine appropriate kpm mapping address and handle any kpm/hme
 * conflicts. Page mapping list and its vcolor parts must be protected.
 */
static caddr_t
sfmmu_kpm_getvaddr(page_t *pp, int *kpm_vac_rangep)
{
        int             vcolor, vcolor_pa;
        caddr_t         vaddr;
        uintptr_t       paddr;


        ASSERT(sfmmu_mlist_held(pp));

        paddr = ptob(pp->p_pagenum);
        vcolor_pa = addr_to_vcolor(paddr);

        if (pp->p_vnode && IS_SWAPFSVP(pp->p_vnode)) {
                vcolor = (PP_NEWPAGE(pp) || PP_ISNC(pp)) ?
                    vcolor_pa : PP_GET_VCOLOR(pp);
        } else {
                vcolor = addr_to_vcolor(pp->p_offset);
        }

        vaddr = kpm_vbase + paddr;
        *kpm_vac_rangep = 0;

        if (vcolor_pa != vcolor) {
                *kpm_vac_rangep = abs(vcolor - vcolor_pa);
                vaddr += ((uintptr_t)(vcolor - vcolor_pa) << MMU_PAGESHIFT);
                vaddr += (vcolor_pa > vcolor) ?
                    ((uintptr_t)vcolor_pa << kpm_size_shift) :
                    ((uintptr_t)(vcolor - vcolor_pa) << kpm_size_shift);

                ASSERT(!PP_ISMAPPED_LARGE(pp));
        }

        if (PP_ISNC(pp))
                return (vaddr);

        if (PP_NEWPAGE(pp)) {
                PP_SET_VCOLOR(pp, vcolor);
                return (vaddr);
        }

        if (PP_GET_VCOLOR(pp) == vcolor)
                return (vaddr);

        ASSERT(!PP_ISMAPPED_KPM(pp));
        sfmmu_kpm_vac_conflict(pp, vaddr);

        return (vaddr);
}

/*
 * VAC conflict state bit values.
 * The following defines are used to make the handling of the
 * various input states more concise. For that the kpm states
 * per kpm_page and per page are combined in a summary state.
 * Each single state has a corresponding bit value in the
 * summary state. These defines only apply for kpm large page
 * mappings. Within comments the abbreviations "kc, c, ks, s"
 * are used as short form of the actual state, e.g. "kc" for
 * "kp_refcntc > 0", etc.
 */
#define KPM_KC  0x00000008      /* kpm_page: kp_refcntc > 0 */
#define KPM_C   0x00000004      /* page: P_KPMC set */
#define KPM_KS  0x00000002      /* kpm_page: kp_refcnts > 0 */
#define KPM_S   0x00000001      /* page: P_KPMS set */

/*
 * Summary states used in sfmmu_kpm_fault (KPM_TSBM_*).
 * See also more detailed comments within in the sfmmu_kpm_fault switch.
 * Abbreviations used:
 * CONFL: VAC conflict(s) within a kpm_page.
 * MAPS:  Mapped small: Page mapped in using a regular page size kpm mapping.
 * RASM:  Re-assembling of a large page mapping possible.
 * RPLS:  Replace: TSB miss due to TSB replacement only.
 * BRKO:  Breakup Other: A large kpm mapping has to be broken because another
 *        page within the kpm_page is already involved in a VAC conflict.
 * BRKT:  Breakup This: A large kpm mapping has to be broken, this page is
 *        is involved in a VAC conflict.
 */
#define KPM_TSBM_CONFL_GONE     (0)
#define KPM_TSBM_MAPS_RASM      (KPM_KS)
#define KPM_TSBM_RPLS_RASM      (KPM_KS | KPM_S)
#define KPM_TSBM_MAPS_BRKO      (KPM_KC)
#define KPM_TSBM_MAPS           (KPM_KC | KPM_KS)
#define KPM_TSBM_RPLS           (KPM_KC | KPM_KS | KPM_S)
#define KPM_TSBM_MAPS_BRKT      (KPM_KC | KPM_C)
#define KPM_TSBM_MAPS_CONFL     (KPM_KC | KPM_C | KPM_KS)
#define KPM_TSBM_RPLS_CONFL     (KPM_KC | KPM_C | KPM_KS | KPM_S)

/*
 * kpm fault handler for mappings with large page size.
 */
int
sfmmu_kpm_fault(caddr_t vaddr, struct memseg *mseg, page_t *pp)
{
        int             error;
        pgcnt_t         inx;
        kpm_page_t      *kp;
        tte_t           tte;
        pfn_t           pfn = pp->p_pagenum;
        kpm_hlk_t       *kpmp;
        kmutex_t        *pml;
        int             alias_range;
        int             uncached = 0;
        kmutex_t        *pmtx;
        int             badstate;
        uint_t          tsbmcase;

        alias_range = IS_KPM_ALIAS_RANGE(vaddr);

        inx = ptokpmp(kpmptop(ptokpmp(pfn)) - mseg->kpm_pbase);
        if (inx >= mseg->kpm_nkpmpgs) {
                cmn_err(CE_PANIC, "sfmmu_kpm_fault: kpm overflow in memseg "
                    "0x%p  pp 0x%p", (void *)mseg, (void *)pp);
        }

        kp = &mseg->kpm_pages[inx];
        kpmp = KPMP_HASH(kp);

        pml = sfmmu_mlist_enter(pp);

        if (!PP_ISMAPPED_KPM(pp)) {
                sfmmu_mlist_exit(pml);
                return (EFAULT);
        }

        mutex_enter(&kpmp->khl_mutex);

        if (alias_range) {
                ASSERT(!PP_ISMAPPED_LARGE(pp));
                if (kp->kp_refcnta > 0) {
                        if (PP_ISKPMC(pp)) {
                                pmtx = sfmmu_page_enter(pp);
                                PP_CLRKPMC(pp);
                                sfmmu_page_exit(pmtx);
                        }
                        /*
                         * Check for vcolor conflicts. Return here
                         * w/ either no conflict (fast path), removed hme
                         * mapping chains (unload conflict) or uncached
                         * (uncache conflict). VACaches are cleaned and
                         * p_vcolor and PP_TNC are set accordingly for the
                         * conflict cases.  Drop kpmp for uncache conflict
                         * cases since it will be grabbed within
                         * sfmmu_kpm_page_cache in case of an uncache
                         * conflict.
                         */
                        mutex_exit(&kpmp->khl_mutex);
                        sfmmu_kpm_vac_conflict(pp, vaddr);
                        mutex_enter(&kpmp->khl_mutex);

                        if (PP_ISNC(pp)) {
                                uncached = 1;
                                pmtx = sfmmu_page_enter(pp);
                                PP_SETKPMC(pp);
                                sfmmu_page_exit(pmtx);
                        }
                        goto smallexit;

                } else {
                        /*
                         * We got a tsbmiss on a not active kpm_page range.
                         * Let segkpm_fault decide how to panic.
                         */
                        error = EFAULT;
                }
                goto exit;
        }

        badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0);
        if (kp->kp_refcntc == -1) {
                /*
                 * We should come here only if trap level tsb miss
                 * handler is disabled.
                 */
                badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 ||
                    PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp));

                if (badstate == 0)
                        goto largeexit;
        }

        if (badstate || kp->kp_refcntc < 0)
                goto badstate_exit;

        /*
         * Combine the per kpm_page and per page kpm VAC states to
         * a summary state in order to make the kpm fault handling
         * more concise.
         */
        tsbmcase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) |
            ((kp->kp_refcnts > 0) ? KPM_KS : 0) |
            (PP_ISKPMC(pp) ? KPM_C : 0) |
            (PP_ISKPMS(pp) ? KPM_S : 0));

        switch (tsbmcase) {
        case KPM_TSBM_CONFL_GONE:               /* - - - - */
                /*
                 * That's fine, we either have no more vac conflict in
                 * this kpm page or someone raced in and has solved the
                 * vac conflict for us -- call sfmmu_kpm_vac_conflict
                 * to take care for correcting the vcolor and flushing
                 * the dcache if required.
                 */
                mutex_exit(&kpmp->khl_mutex);
                sfmmu_kpm_vac_conflict(pp, vaddr);
                mutex_enter(&kpmp->khl_mutex);

                if (PP_ISNC(pp) || kp->kp_refcnt <= 0 ||
                    addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
                        panic("sfmmu_kpm_fault: inconsistent CONFL_GONE "
                            "state, pp=%p", (void *)pp);
                }
                goto largeexit;

        case KPM_TSBM_MAPS_RASM:                /* - - ks - */
                /*
                 * All conflicts in this kpm page are gone but there are
                 * already small mappings around, so we also map this
                 * page small. This could be the trigger case for a
                 * small mapping reaper, if this is really needed.
                 * For now fall thru to the KPM_TSBM_MAPS handling.
                 */

        case KPM_TSBM_MAPS:                     /* kc - ks - */
                /*
                 * Large page mapping is already broken, this page is not
                 * conflicting, so map it small. Call sfmmu_kpm_vac_conflict
                 * to take care for correcting the vcolor and flushing
                 * the dcache if required.
                 */
                mutex_exit(&kpmp->khl_mutex);
                sfmmu_kpm_vac_conflict(pp, vaddr);
                mutex_enter(&kpmp->khl_mutex);

                if (PP_ISNC(pp) || kp->kp_refcnt <= 0 ||
                    addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
                        panic("sfmmu_kpm_fault:  inconsistent MAPS state, "
                            "pp=%p", (void *)pp);
                }
                kp->kp_refcnt--;
                kp->kp_refcnts++;
                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMS(pp);
                sfmmu_page_exit(pmtx);
                goto smallexit;

        case KPM_TSBM_RPLS_RASM:                /* - - ks s */
                /*
                 * All conflicts in this kpm page are gone but this page
                 * is mapped small. This could be the trigger case for a
                 * small mapping reaper, if this is really needed.
                 * For now we drop it in small again. Fall thru to the
                 * KPM_TSBM_RPLS handling.
                 */

        case KPM_TSBM_RPLS:                     /* kc - ks s */
                /*
                 * Large page mapping is already broken, this page is not
                 * conflicting but already mapped small, so drop it in
                 * small again.
                 */
                if (PP_ISNC(pp) ||
                    addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
                        panic("sfmmu_kpm_fault:  inconsistent RPLS state, "
                            "pp=%p", (void *)pp);
                }
                goto smallexit;

        case KPM_TSBM_MAPS_BRKO:                /* kc - - - */
                /*
                 * The kpm page where we live in is marked conflicting
                 * but this page is not conflicting. So we have to map it
                 * in small. Call sfmmu_kpm_vac_conflict to take care for
                 * correcting the vcolor and flushing the dcache if required.
                 */
                mutex_exit(&kpmp->khl_mutex);
                sfmmu_kpm_vac_conflict(pp, vaddr);
                mutex_enter(&kpmp->khl_mutex);

                if (PP_ISNC(pp) || kp->kp_refcnt <= 0 ||
                    addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
                        panic("sfmmu_kpm_fault:  inconsistent MAPS_BRKO state, "
                            "pp=%p", (void *)pp);
                }
                kp->kp_refcnt--;
                kp->kp_refcnts++;
                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMS(pp);
                sfmmu_page_exit(pmtx);
                goto smallexit;

        case KPM_TSBM_MAPS_BRKT:                /* kc c - - */
        case KPM_TSBM_MAPS_CONFL:               /* kc c ks - */
                if (!PP_ISMAPPED(pp)) {
                        /*
                         * We got a tsbmiss on kpm large page range that is
                         * marked to contain vac conflicting pages introduced
                         * by hme mappings. The hme mappings are all gone and
                         * must have bypassed the kpm alias prevention logic.
                         */
                        panic("sfmmu_kpm_fault: stale VAC conflict, pp=%p",
                            (void *)pp);
                }

                /*
                 * Check for vcolor conflicts. Return here w/ either no
                 * conflict (fast path), removed hme mapping chains
                 * (unload conflict) or uncached (uncache conflict).
                 * Dcache is cleaned and p_vcolor and P_TNC are set
                 * accordingly. Drop kpmp for uncache conflict cases
                 * since it will be grabbed within sfmmu_kpm_page_cache
                 * in case of an uncache conflict.
                 */
                mutex_exit(&kpmp->khl_mutex);
                sfmmu_kpm_vac_conflict(pp, vaddr);
                mutex_enter(&kpmp->khl_mutex);

                if (kp->kp_refcnt <= 0)
                        panic("sfmmu_kpm_fault: bad refcnt kp=%p", (void *)kp);

                if (PP_ISNC(pp)) {
                        uncached = 1;
                } else {
                        /*
                         * When an unload conflict is solved and there are
                         * no other small mappings around, we can resume
                         * largepage mode. Otherwise we have to map or drop
                         * in small. This could be a trigger for a small
                         * mapping reaper when this was the last conflict
                         * within the kpm page and when there are only
                         * other small mappings around.
                         */
                        ASSERT(addr_to_vcolor(vaddr) == PP_GET_VCOLOR(pp));
                        ASSERT(kp->kp_refcntc > 0);
                        kp->kp_refcntc--;
                        pmtx = sfmmu_page_enter(pp);
                        PP_CLRKPMC(pp);
                        sfmmu_page_exit(pmtx);
                        ASSERT(PP_ISKPMS(pp) == 0);
                        if (kp->kp_refcntc == 0 && kp->kp_refcnts == 0)
                                goto largeexit;
                }

                kp->kp_refcnt--;
                kp->kp_refcnts++;
                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMS(pp);
                sfmmu_page_exit(pmtx);
                goto smallexit;

        case KPM_TSBM_RPLS_CONFL:               /* kc c ks s */
                if (!PP_ISMAPPED(pp)) {
                        /*
                         * We got a tsbmiss on kpm large page range that is
                         * marked to contain vac conflicting pages introduced
                         * by hme mappings. They are all gone and must have
                         * somehow bypassed the kpm alias prevention logic.
                         */
                        panic("sfmmu_kpm_fault: stale VAC conflict, pp=%p",
                            (void *)pp);
                }

                /*
                 * This state is only possible for an uncached mapping.
                 */
                if (!PP_ISNC(pp)) {
                        panic("sfmmu_kpm_fault: page not uncached, pp=%p",
                            (void *)pp);
                }
                uncached = 1;
                goto smallexit;

        default:
badstate_exit:
                panic("sfmmu_kpm_fault: inconsistent VAC state, vaddr=%p kp=%p "
                    "pp=%p", (void *)vaddr, (void *)kp, (void *)pp);
        }

smallexit:
        /* tte assembly */
        if (uncached == 0)
                KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
        else
                KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);

        /* tsb dropin */
        sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);

        error = 0;
        goto exit;

largeexit:
        if (kp->kp_refcnt > 0) {

                /* tte assembly */
                KPM_TTE_VCACHED(tte.ll, pfn, TTE4M);

                /* tsb dropin */
                sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT4M);

                if (kp->kp_refcntc == 0) {
                        /* Set "go" flag for TL tsbmiss handler */
                        sfmmu_kpm_tsbmtl(&kp->kp_refcntc, &kpmp->khl_lock,
                            KPMTSBM_START);
                }
                ASSERT(kp->kp_refcntc == -1);
                error = 0;

        } else
                error = EFAULT;
exit:
        mutex_exit(&kpmp->khl_mutex);
        sfmmu_mlist_exit(pml);
        return (error);
}

/*
 * kpm fault handler for mappings with small page size.
 */
int
sfmmu_kpm_fault_small(caddr_t vaddr, struct memseg *mseg, page_t *pp)
{
        int             error = 0;
        pgcnt_t         inx;
        kpm_spage_t     *ksp;
        kpm_shlk_t      *kpmsp;
        kmutex_t        *pml;
        pfn_t           pfn = pp->p_pagenum;
        tte_t           tte;
        kmutex_t        *pmtx;
        int             oldval;

        inx = pfn - mseg->kpm_pbase;
        ksp = &mseg->kpm_spages[inx];
        kpmsp = KPMP_SHASH(ksp);

        pml = sfmmu_mlist_enter(pp);

        if (!PP_ISMAPPED_KPM(pp)) {
                sfmmu_mlist_exit(pml);
                return (EFAULT);
        }

        /*
         * kp_mapped lookup protected by mlist mutex
         */
        if (ksp->kp_mapped == KPM_MAPPEDS) {
                /*
                 * Fast path tsbmiss
                 */
                ASSERT(!PP_ISKPMC(pp));
                ASSERT(!PP_ISNC(pp));

                /* tte assembly */
                KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);

                /* tsb dropin */
                sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);

        } else if (ksp->kp_mapped == KPM_MAPPEDSC) {
                /*
                 * Got here due to existing or gone kpm/hme VAC conflict.
                 * Recheck for vcolor conflicts. Return here w/ either
                 * no conflict, removed hme mapping chain (unload
                 * conflict) or uncached (uncache conflict). VACaches
                 * are cleaned and p_vcolor and PP_TNC are set accordingly
                 * for the conflict cases.
                 */
                sfmmu_kpm_vac_conflict(pp, vaddr);

                if (PP_ISNC(pp)) {
                        /* ASSERT(pp->p_share); XXX use hat_page_getshare */

                        /* tte assembly */
                        KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);

                        /* tsb dropin */
                        sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);

                        oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
                            &kpmsp->kshl_lock, (KPM_MAPPED_GO | KPM_MAPPEDSC));

                        if (oldval != KPM_MAPPEDSC)
                                panic("sfmmu_kpm_fault_small: "
                                    "stale smallpages mapping");
                } else {
                        if (PP_ISKPMC(pp)) {
                                pmtx = sfmmu_page_enter(pp);
                                PP_CLRKPMC(pp);
                                sfmmu_page_exit(pmtx);
                        }

                        /* tte assembly */
                        KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);

                        /* tsb dropin */
                        sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);

                        oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
                            &kpmsp->kshl_lock, (KPM_MAPPED_GO | KPM_MAPPEDS));

                        if (oldval != KPM_MAPPEDSC)
                                panic("sfmmu_kpm_fault_small: "
                                    "stale smallpages mapping");
                }

        } else {
                /*
                 * We got a tsbmiss on a not active kpm_page range.
                 * Let decide segkpm_fault how to panic.
                 */
                error = EFAULT;
        }

        sfmmu_mlist_exit(pml);
        return (error);
}

/*
 * Check/handle potential hme/kpm mapping conflicts
 */
static void
sfmmu_kpm_vac_conflict(page_t *pp, caddr_t vaddr)
{
        int             vcolor;
        struct sf_hment *sfhmep;
        struct hat      *tmphat;
        struct sf_hment *tmphme = NULL;
        struct hme_blk  *hmeblkp;
        tte_t           tte;

        ASSERT(sfmmu_mlist_held(pp));

        if (PP_ISNC(pp))
                return;

        vcolor = addr_to_vcolor(vaddr);
        if (PP_GET_VCOLOR(pp) == vcolor)
                return;

        /*
         * There could be no vcolor conflict between a large cached
         * hme page and a non alias range kpm page (neither large nor
         * small mapped). So if a hme conflict already exists between
         * a constituent page of a large hme mapping and a shared small
         * conflicting hme mapping, both mappings must be already
         * uncached at this point.
         */
        ASSERT(!PP_ISMAPPED_LARGE(pp));

        if (!PP_ISMAPPED(pp)) {
                /*
                 * Previous hme user of page had a different color
                 * but since there are no current users
                 * we just flush the cache and change the color.
                 */
                SFMMU_STAT(sf_pgcolor_conflict);
                sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp));
                PP_SET_VCOLOR(pp, vcolor);
                return;
        }

        /*
         * If we get here we have a vac conflict with a current hme
         * mapping. This must have been established by forcing a wrong
         * colored mapping, e.g. by using mmap(2) with MAP_FIXED.
         */

        /*
         * Check if any mapping is in same as or if it is locked
         * since in that case we need to uncache.
         */
        for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) {
                tmphme = sfhmep->hme_next;
                if (IS_PAHME(sfhmep))
                        continue;
                hmeblkp = sfmmu_hmetohblk(sfhmep);
                tmphat = hblktosfmmu(hmeblkp);
                sfmmu_copytte(&sfhmep->hme_tte, &tte);
                ASSERT(TTE_IS_VALID(&tte));
                if ((tmphat == ksfmmup) || hmeblkp->hblk_lckcnt) {
                        /*
                         * We have an uncache conflict
                         */
                        SFMMU_STAT(sf_uncache_conflict);
                        sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1);
                        return;
                }
        }

        /*
         * We have an unload conflict
         */
        SFMMU_STAT(sf_unload_conflict);

        for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) {
                tmphme = sfhmep->hme_next;
                if (IS_PAHME(sfhmep))
                        continue;
                hmeblkp = sfmmu_hmetohblk(sfhmep);
                (void) sfmmu_pageunload(pp, sfhmep, TTE8K);
        }

        /*
         * Unloads only does tlb flushes so we need to flush the
         * dcache vcolor here.
         */
        sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp));
        PP_SET_VCOLOR(pp, vcolor);
}

/*
 * Remove all kpm mappings using kpme's for pp and check that
 * all kpm mappings (w/ and w/o kpme's) are gone.
 */
void
sfmmu_kpm_pageunload(page_t *pp)
{
        caddr_t         vaddr;
        struct kpme     *kpme, *nkpme;

        ASSERT(pp != NULL);
        ASSERT(pp->p_kpmref);
        ASSERT(sfmmu_mlist_held(pp));

        vaddr = hat_kpm_page2va(pp, 1);

        for (kpme = pp->p_kpmelist; kpme; kpme = nkpme) {
                ASSERT(kpme->kpe_page == pp);

                if (pp->p_kpmref == 0)
                        panic("sfmmu_kpm_pageunload: stale p_kpmref pp=%p "
                            "kpme=%p", (void *)pp, (void *)kpme);

                nkpme = kpme->kpe_next;

                /* Add instance callback here here if needed later */
                sfmmu_kpme_sub(kpme, pp);
        }

        /*
         * Also correct after mixed kpme/nonkpme mappings. If nonkpme
         * segkpm clients have unlocked the page and forgot to mapout
         * we panic here.
         */
        if (pp->p_kpmref != 0)
                panic("sfmmu_kpm_pageunload: bad refcnt pp=%p", (void *)pp);

        sfmmu_kpm_mapout(pp, vaddr);
}

/*
 * Remove a large kpm mapping from kernel TSB and all TLB's.
 */
static void
sfmmu_kpm_demap_large(caddr_t vaddr)
{
        sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M);
        sfmmu_kpm_demap_tlbs(vaddr);
}

/*
 * Remove a small kpm mapping from kernel TSB and all TLB's.
 */
static void
sfmmu_kpm_demap_small(caddr_t vaddr)
{
        sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT);
        sfmmu_kpm_demap_tlbs(vaddr);
}

/*
 * Demap a kpm mapping in all TLB's.
 */
static void
sfmmu_kpm_demap_tlbs(caddr_t vaddr)
{
        cpuset_t cpuset;

        kpreempt_disable();
        cpuset = ksfmmup->sfmmu_cpusran;
        CPUSET_AND(cpuset, cpu_ready_set);
        CPUSET_DEL(cpuset, CPU->cpu_id);
        SFMMU_XCALL_STATS(ksfmmup);

        xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)vaddr,
            (uint64_t)ksfmmup);
        vtag_flushpage(vaddr, (uint64_t)ksfmmup);

        kpreempt_enable();
}

/*
 * Summary states used in sfmmu_kpm_vac_unload (KPM_VUL__*).
 * See also more detailed comments within in the sfmmu_kpm_vac_unload switch.
 * Abbreviations used:
 * BIG:   Large page kpm mapping in use.
 * CONFL: VAC conflict(s) within a kpm_page.
 * INCR:  Count of conflicts within a kpm_page is going to be incremented.
 * DECR:  Count of conflicts within a kpm_page is going to be decremented.
 * UNMAP_SMALL: A small (regular page size) mapping is going to be unmapped.
 * TNC:   Temporary non cached: a kpm mapped page is mapped in TNC state.
 */
#define KPM_VUL_BIG             (0)
#define KPM_VUL_CONFL_INCR1     (KPM_KS)
#define KPM_VUL_UNMAP_SMALL1    (KPM_KS | KPM_S)
#define KPM_VUL_CONFL_INCR2     (KPM_KC)
#define KPM_VUL_CONFL_INCR3     (KPM_KC | KPM_KS)
#define KPM_VUL_UNMAP_SMALL2    (KPM_KC | KPM_KS | KPM_S)
#define KPM_VUL_CONFL_DECR1     (KPM_KC | KPM_C)
#define KPM_VUL_CONFL_DECR2     (KPM_KC | KPM_C | KPM_KS)
#define KPM_VUL_TNC             (KPM_KC | KPM_C | KPM_KS | KPM_S)

/*
 * Handle VAC unload conflicts introduced by hme mappings or vice
 * versa when a hme conflict mapping is replaced by a non conflict
 * one. Perform actions and state transitions according to the
 * various page and kpm_page entry states. VACache flushes are in
 * the responsibiliy of the caller. We still hold the mlist lock.
 */
void
sfmmu_kpm_vac_unload(page_t *pp, caddr_t vaddr)
{
        kpm_page_t      *kp;
        kpm_hlk_t       *kpmp;
        caddr_t         kpmvaddr = hat_kpm_page2va(pp, 1);
        int             newcolor;
        kmutex_t        *pmtx;
        uint_t          vacunlcase;
        int             badstate = 0;
        kpm_spage_t     *ksp;
        kpm_shlk_t      *kpmsp;

        ASSERT(PAGE_LOCKED(pp));
        ASSERT(sfmmu_mlist_held(pp));
        ASSERT(!PP_ISNC(pp));

        newcolor = addr_to_vcolor(kpmvaddr) != addr_to_vcolor(vaddr);
        if (kpm_smallpages)
                goto smallpages_vac_unload;

        PP2KPMPG(pp, kp);
        kpmp = KPMP_HASH(kp);
        mutex_enter(&kpmp->khl_mutex);

        if (IS_KPM_ALIAS_RANGE(kpmvaddr)) {
                if (kp->kp_refcnta < 1) {
                        panic("sfmmu_kpm_vac_unload: bad refcnta kpm_page=%p\n",
                            (void *)kp);
                }

                if (PP_ISKPMC(pp) == 0) {
                        if (newcolor == 0)
                                goto exit;
                        sfmmu_kpm_demap_small(kpmvaddr);
                        pmtx = sfmmu_page_enter(pp);
                        PP_SETKPMC(pp);
                        sfmmu_page_exit(pmtx);

                } else if (newcolor == 0) {
                        pmtx = sfmmu_page_enter(pp);
                        PP_CLRKPMC(pp);
                        sfmmu_page_exit(pmtx);

                } else {
                        badstate++;
                }

                goto exit;
        }

        badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0);
        if (kp->kp_refcntc == -1) {
                /*
                 * We should come here only if trap level tsb miss
                 * handler is disabled.
                 */
                badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 ||
                    PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp));
        } else {
                badstate |= (kp->kp_refcntc < 0);
        }

        if (badstate)
                goto exit;

        if (PP_ISKPMC(pp) == 0 && newcolor == 0) {
                ASSERT(PP_ISKPMS(pp) == 0);
                goto exit;
        }

        /*
         * Combine the per kpm_page and per page kpm VAC states
         * to a summary state in order to make the vac unload
         * handling more concise.
         */
        vacunlcase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) |
            ((kp->kp_refcnts > 0) ? KPM_KS : 0) |
            (PP_ISKPMC(pp) ? KPM_C : 0) |
            (PP_ISKPMS(pp) ? KPM_S : 0));

        switch (vacunlcase) {
        case KPM_VUL_BIG:                               /* - - - - */
                /*
                 * Have to breakup the large page mapping to be
                 * able to handle the conflicting hme vaddr.
                 */
                if (kp->kp_refcntc == -1) {
                        /* remove go indication */
                        sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
                            &kpmp->khl_lock, KPMTSBM_STOP);
                }
                sfmmu_kpm_demap_large(kpmvaddr);

                ASSERT(kp->kp_refcntc == 0);
                kp->kp_refcntc++;
                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMC(pp);
                sfmmu_page_exit(pmtx);
                break;

        case KPM_VUL_UNMAP_SMALL1:                      /* -  - ks s */
        case KPM_VUL_UNMAP_SMALL2:                      /* kc - ks s */
                /*
                 * New conflict w/ an active kpm page, actually mapped
                 * in by small TSB/TLB entries. Remove the mapping and
                 * update states.
                 */
                ASSERT(newcolor);
                sfmmu_kpm_demap_small(kpmvaddr);
                kp->kp_refcnts--;
                kp->kp_refcnt++;
                kp->kp_refcntc++;
                pmtx = sfmmu_page_enter(pp);
                PP_CLRKPMS(pp);
                PP_SETKPMC(pp);
                sfmmu_page_exit(pmtx);
                break;

        case KPM_VUL_CONFL_INCR1:                       /* -  - ks - */
        case KPM_VUL_CONFL_INCR2:                       /* kc - -  - */
        case KPM_VUL_CONFL_INCR3:                       /* kc - ks - */
                /*
                 * New conflict on a active kpm mapped page not yet in
                 * TSB/TLB. Mark page and increment the kpm_page conflict
                 * count.
                 */
                ASSERT(newcolor);
                kp->kp_refcntc++;
                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMC(pp);
                sfmmu_page_exit(pmtx);
                break;

        case KPM_VUL_CONFL_DECR1:                       /* kc c -  - */
        case KPM_VUL_CONFL_DECR2:                       /* kc c ks - */
                /*
                 * A conflicting hme mapping is removed for an active
                 * kpm page not yet in TSB/TLB. Unmark page and decrement
                 * the kpm_page conflict count.
                 */
                ASSERT(newcolor == 0);
                kp->kp_refcntc--;
                pmtx = sfmmu_page_enter(pp);
                PP_CLRKPMC(pp);
                sfmmu_page_exit(pmtx);
                break;

        case KPM_VUL_TNC:                               /* kc c ks s */
                cmn_err(CE_NOTE, "sfmmu_kpm_vac_unload: "
                    "page not in NC state");
                /* FALLTHRU */

        default:
                badstate++;
        }
exit:
        if (badstate) {
                panic("sfmmu_kpm_vac_unload: inconsistent VAC state, "
                    "kpmvaddr=%p kp=%p pp=%p",
                    (void *)kpmvaddr, (void *)kp, (void *)pp);
        }
        mutex_exit(&kpmp->khl_mutex);

        return;

smallpages_vac_unload:
        if (newcolor == 0)
                return;

        PP2KPMSPG(pp, ksp);
        kpmsp = KPMP_SHASH(ksp);

        if (PP_ISKPMC(pp) == 0) {
                if (ksp->kp_mapped == KPM_MAPPEDS) {
                        /*
                         * Stop TL tsbmiss handling
                         */
                        (void) sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
                            &kpmsp->kshl_lock, KPM_MAPPEDSC);

                        sfmmu_kpm_demap_small(kpmvaddr);

                } else if (ksp->kp_mapped != KPM_MAPPEDSC) {
                        panic("sfmmu_kpm_vac_unload: inconsistent mapping");
                }

                pmtx = sfmmu_page_enter(pp);
                PP_SETKPMC(pp);
                sfmmu_page_exit(pmtx);

        } else {
                if (ksp->kp_mapped != KPM_MAPPEDSC)
                        panic("sfmmu_kpm_vac_unload: inconsistent mapping");
        }
}

/*
 * Page is marked to be in VAC conflict to an existing kpm mapping
 * or is kpm mapped using only the regular pagesize. Called from
 * sfmmu_hblk_unload when a mlist is completely removed.
 */
void
sfmmu_kpm_hme_unload(page_t *pp)
{
        /* tte assembly */
        kpm_page_t      *kp;
        kpm_hlk_t       *kpmp;
        caddr_t         vaddr;
        kmutex_t        *pmtx;
        uint_t          flags;
        kpm_spage_t     *ksp;

        ASSERT(sfmmu_mlist_held(pp));
        ASSERT(PP_ISMAPPED_KPM(pp));

        flags = pp->p_nrm & (P_KPMC | P_KPMS);
        if (kpm_smallpages)
                goto smallpages_hme_unload;

        if (flags == (P_KPMC | P_KPMS)) {
                panic("sfmmu_kpm_hme_unload: page should be uncached");

        } else if (flags == P_KPMS) {
                /*
                 * Page mapped small but not involved in VAC conflict
                 */
                return;
        }

        vaddr = hat_kpm_page2va(pp, 1);

        PP2KPMPG(pp, kp);
        kpmp = KPMP_HASH(kp);
        mutex_enter(&kpmp->khl_mutex);

        if (IS_KPM_ALIAS_RANGE(vaddr)) {
                if (kp->kp_refcnta < 1) {
                        panic("sfmmu_kpm_hme_unload: bad refcnta kpm_page=%p\n",
                            (void *)kp);
                }
        } else {
                if (kp->kp_refcntc < 1) {
                        panic("sfmmu_kpm_hme_unload: bad refcntc kpm_page=%p\n",
                            (void *)kp);
                }
                kp->kp_refcntc--;
        }

        pmtx = sfmmu_page_enter(pp);
        PP_CLRKPMC(pp);
        sfmmu_page_exit(pmtx);

        mutex_exit(&kpmp->khl_mutex);
        return;

smallpages_hme_unload:
        if (flags != P_KPMC)
                panic("sfmmu_kpm_hme_unload: page should be uncached");

        vaddr = hat_kpm_page2va(pp, 1);
        PP2KPMSPG(pp, ksp);

        if (ksp->kp_mapped != KPM_MAPPEDSC)
                panic("sfmmu_kpm_hme_unload: inconsistent mapping");

        /*
         * Keep KPM_MAPPEDSC until the next kpm tsbmiss where it
         * prevents TL tsbmiss handling and force a hat_kpm_fault.
         * There we can start over again.
         */

        pmtx = sfmmu_page_enter(pp);
        PP_CLRKPMC(pp);
        sfmmu_page_exit(pmtx);
}

/*
 * Special hooks for sfmmu_page_cache_array() when changing the
 * cacheability of a page. It is used to obey the hat_kpm lock
 * ordering (mlist -> kpmp -> spl, and back).
 */
kpm_hlk_t *
sfmmu_kpm_kpmp_enter(page_t *pp, pgcnt_t npages)
{
        kpm_page_t      *kp;
        kpm_hlk_t       *kpmp;

        ASSERT(sfmmu_mlist_held(pp));

        if (kpm_smallpages || PP_ISMAPPED_KPM(pp) == 0)
                return (NULL);

        ASSERT(npages <= kpmpnpgs);

        PP2KPMPG(pp, kp);
        kpmp = KPMP_HASH(kp);
        mutex_enter(&kpmp->khl_mutex);

        return (kpmp);
}

void
sfmmu_kpm_kpmp_exit(kpm_hlk_t *kpmp)
{
        if (kpm_smallpages || kpmp == NULL)
                return;

        mutex_exit(&kpmp->khl_mutex);
}

/*
 * Summary states used in sfmmu_kpm_page_cache (KPM_*).
 * See also more detailed comments within in the sfmmu_kpm_page_cache switch.
 * Abbreviations used:
 * UNC:     Input state for an uncache request.
 *   BIG:     Large page kpm mapping in use.
 *   SMALL:   Page has a small kpm mapping within a kpm_page range.
 *   NODEMAP: No demap needed.
 *   NOP:     No operation needed on this input state.
 * CACHE:   Input state for a re-cache request.
 *   MAPS:    Page is in TNC and kpm VAC conflict state and kpm mapped small.
 *   NOMAP:   Page is in TNC and kpm VAC conflict state, but not small kpm
 *            mapped.
 *   NOMAPO:  Page is in TNC and kpm VAC conflict state, but not small kpm
 *            mapped. There are also other small kpm mappings within this
 *            kpm_page.
 */
#define KPM_UNC_BIG             (0)
#define KPM_UNC_NODEMAP1        (KPM_KS)
#define KPM_UNC_SMALL1          (KPM_KS | KPM_S)
#define KPM_UNC_NODEMAP2        (KPM_KC)
#define KPM_UNC_NODEMAP3        (KPM_KC | KPM_KS)
#define KPM_UNC_SMALL2          (KPM_KC | KPM_KS | KPM_S)
#define KPM_UNC_NOP1            (KPM_KC | KPM_C)
#define KPM_UNC_NOP2            (KPM_KC | KPM_C | KPM_KS)
#define KPM_CACHE_NOMAP         (KPM_KC | KPM_C)
#define KPM_CACHE_NOMAPO        (KPM_KC | KPM_C | KPM_KS)
#define KPM_CACHE_MAPS          (KPM_KC | KPM_C | KPM_KS | KPM_S)

/*
 * This function is called when the virtual cacheability of a page
 * is changed and the page has an actice kpm mapping. The mlist mutex,
 * the spl hash lock and the kpmp mutex (if needed) are already grabbed.
 */
/*ARGSUSED2*/
void
sfmmu_kpm_page_cache(page_t *pp, int flags, int cache_flush_tag)
{
        kpm_page_t      *kp;
        kpm_hlk_t       *kpmp;
        caddr_t         kpmvaddr;
        int             badstate = 0;
        uint_t          pgcacase;
        kpm_spage_t     *ksp;
        kpm_shlk_t      *kpmsp;
        int             oldval;

        ASSERT(PP_ISMAPPED_KPM(pp));
        ASSERT(sfmmu_mlist_held(pp));
        ASSERT(sfmmu_page_spl_held(pp));

        if (flags != HAT_TMPNC && flags != HAT_CACHE)
                panic("sfmmu_kpm_page_cache: bad flags");

        kpmvaddr = hat_kpm_page2va(pp, 1);

        if (flags == HAT_TMPNC && cache_flush_tag == CACHE_FLUSH) {
                pfn_t pfn = pp->p_pagenum;
                int vcolor = addr_to_vcolor(kpmvaddr);
                cpuset_t cpuset = cpu_ready_set;

                /* Flush vcolor in DCache */
                CPUSET_DEL(cpuset, CPU->cpu_id);
                SFMMU_XCALL_STATS(ksfmmup);
                xt_some(cpuset, vac_flushpage_tl1, pfn, vcolor);
                vac_flushpage(pfn, vcolor);
        }

        if (kpm_smallpages)
                goto smallpages_page_cache;

        PP2KPMPG(pp, kp);
        kpmp = KPMP_HASH(kp);
        ASSERT(MUTEX_HELD(&kpmp->khl_mutex));

        if (IS_KPM_ALIAS_RANGE(kpmvaddr)) {
                if (kp->kp_refcnta < 1) {
                        panic("sfmmu_kpm_page_cache: bad refcnta "
                            "kpm_page=%p\n", (void *)kp);
                }
                sfmmu_kpm_demap_small(kpmvaddr);
                if (flags == HAT_TMPNC) {
                        PP_SETKPMC(pp);
                        ASSERT(!PP_ISKPMS(pp));
                } else {
                        ASSERT(PP_ISKPMC(pp));
                        PP_CLRKPMC(pp);
                }
                goto exit;
        }

        badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0);
        if (kp->kp_refcntc == -1) {
                /*
                 * We should come here only if trap level tsb miss
                 * handler is disabled.
                 */
                badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 ||
                    PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp));
        } else {
                badstate |= (kp->kp_refcntc < 0);
        }

        if (badstate)
                goto exit;

        /*
         * Combine the per kpm_page and per page kpm VAC states to
         * a summary state in order to make the VAC cache/uncache
         * handling more concise.
         */
        pgcacase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) |
            ((kp->kp_refcnts > 0) ? KPM_KS : 0) |
            (PP_ISKPMC(pp) ? KPM_C : 0) |
            (PP_ISKPMS(pp) ? KPM_S : 0));

        if (flags == HAT_CACHE) {
                switch (pgcacase) {
                case KPM_CACHE_MAPS:                    /* kc c ks s */
                        sfmmu_kpm_demap_small(kpmvaddr);
                        if (kp->kp_refcnts < 1) {
                                panic("sfmmu_kpm_page_cache: bad refcnts "
                                "kpm_page=%p\n", (void *)kp);
                        }
                        kp->kp_refcnts--;
                        kp->kp_refcnt++;
                        PP_CLRKPMS(pp);
                        /* FALLTHRU */

                case KPM_CACHE_NOMAP:                   /* kc c -  - */
                case KPM_CACHE_NOMAPO:                  /* kc c ks - */
                        kp->kp_refcntc--;
                        PP_CLRKPMC(pp);
                        break;

                default:
                        badstate++;
                }
                goto exit;
        }

        switch (pgcacase) {
        case KPM_UNC_BIG:                               /* - - - - */
                if (kp->kp_refcnt < 1) {
                        panic("sfmmu_kpm_page_cache: bad refcnt "
                            "kpm_page=%p\n", (void *)kp);
                }

                /*
                 * Have to breakup the large page mapping in preparation
                 * to the upcoming TNC mode handled by small mappings.
                 * The demap can already be done due to another conflict
                 * within the kpm_page.
                 */
                if (kp->kp_refcntc == -1) {
                        /* remove go indication */
                        sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
                            &kpmp->khl_lock, KPMTSBM_STOP);
                }
                ASSERT(kp->kp_refcntc == 0);
                sfmmu_kpm_demap_large(kpmvaddr);
                kp->kp_refcntc++;
                PP_SETKPMC(pp);
                break;

        case KPM_UNC_SMALL1:                            /* -  - ks s */
        case KPM_UNC_SMALL2:                            /* kc - ks s */
                /*
                 * Have to demap an already small kpm mapping in preparation
                 * to the upcoming TNC mode. The demap can already be done
                 * due to another conflict within the kpm_page.
                 */
                sfmmu_kpm_demap_small(kpmvaddr);
                kp->kp_refcntc++;
                kp->kp_refcnts--;
                kp->kp_refcnt++;
                PP_CLRKPMS(pp);
                PP_SETKPMC(pp);
                break;

        case KPM_UNC_NODEMAP1:                          /* -  - ks - */
                /* fallthru */

        case KPM_UNC_NODEMAP2:                          /* kc - -  - */
        case KPM_UNC_NODEMAP3:                          /* kc - ks - */
                kp->kp_refcntc++;
                PP_SETKPMC(pp);
                break;

        case KPM_UNC_NOP1:                              /* kc c -  - */
        case KPM_UNC_NOP2:                              /* kc c ks - */
                break;

        default:
                badstate++;
        }
exit:
        if (badstate) {
                panic("sfmmu_kpm_page_cache: inconsistent VAC state "
                    "kpmvaddr=%p kp=%p pp=%p", (void *)kpmvaddr,
                    (void *)kp, (void *)pp);
        }
        return;

smallpages_page_cache:
        PP2KPMSPG(pp, ksp);
        kpmsp = KPMP_SHASH(ksp);

        /*
         * marked as nogo for we will fault in and resolve it
         * through sfmmu_kpm_fault_small
         */
        oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag, &kpmsp->kshl_lock,
            KPM_MAPPEDSC);

        if (!(oldval == KPM_MAPPEDS || oldval == KPM_MAPPEDSC))
                panic("smallpages_page_cache: inconsistent mapping");

        sfmmu_kpm_demap_small(kpmvaddr);

        if (flags == HAT_TMPNC) {
                PP_SETKPMC(pp);
                ASSERT(!PP_ISKPMS(pp));

        } else {
                ASSERT(PP_ISKPMC(pp));
                PP_CLRKPMC(pp);
        }

        /*
         * Keep KPM_MAPPEDSC until the next kpm tsbmiss where it
         * prevents TL tsbmiss handling and force a hat_kpm_fault.
         * There we can start over again.
         */
}