/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * sun4u Memory Scrubbing
 *
 * On detection of a correctable memory ECC error, the sun4u kernel
 * returns the corrected data to the requester and re-writes it
 * to memory (DRAM).  So if the correctable error was transient,
 * the read has effectively been cleaned (scrubbed) from memory.
 *
 * Scrubbing thus reduces the likelyhood that multiple transient errors
 * will occur in the same memory word, making uncorrectable errors due
 * to transients less likely.
 *
 * Thus is born the desire that every memory location be periodically
 * accessed.
 *
 * This file implements a memory scrubbing thread.  This scrubber
 * guarantees that all of physical memory is accessed periodically
 * (memscrub_period_sec -- 12 hours).
 *
 * It attempts to do this as unobtrusively as possible.  The thread
 * schedules itself to wake up at an interval such that if it reads
 * memscrub_span_pages (32MB) on each wakeup, it will read all of physical
 * memory in in memscrub_period_sec (12 hours).
 *
 * The scrubber uses the block load and prefetch hardware to read memory
 * @ 1300MB/s, so it reads spans of 32MB in 0.025 seconds.  Unlike the
 * original sun4d scrubber the sun4u scrubber does not read ahead if the
 * system is idle because we can read memory very efficently.
 *
 * The scrubber maintains a private copy of the phys_install memory list
 * to keep track of what memory should be scrubbed.
 *
 * The global routines memscrub_add_span() and memscrub_delete_span() are
 * used to add and delete from this list.  If hotplug memory is later
 * supported these two routines can be used to notify the scrubber of
 * memory configuration changes.
 *
 * The following parameters can be set via /etc/system
 *
 * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (8MB)
 * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours)
 * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (MINCLSYSPRI)
 * memscrub_delay_start_sec = (5 minutes)
 * memscrub_verbose = (0)
 * memscrub_override_ticks = (1 tick)
 * disable_memscrub = (0)
 * pause_memscrub = (0)
 * read_all_memscrub = (0)
 *
 * The scrubber will print NOTICE messages of what it is doing if
 * "memscrub_verbose" is set.
 *
 * If the scrubber's sleep time calculation drops to zero ticks,
 * memscrub_override_ticks will be used as the sleep time instead. The
 * sleep time should only drop to zero on a system with over 131.84
 * terabytes of memory, or where the default scrubber parameters have
 * been adjusted. For example, reducing memscrub_span_pages or
 * memscrub_period_sec causes the sleep time to drop to zero with less
 * memory. Note that since the sleep time is calculated in clock ticks,
 * using hires clock ticks allows for more memory before the sleep time
 * becomes zero.
 *
 * The scrubber will exit (or never be started) if it finds the variable
 * "disable_memscrub" set.
 *
 * The scrubber will pause (not read memory) when "pause_memscrub"
 * is set.  It will check the state of pause_memscrub at each wakeup
 * period.  The scrubber will not make up for lost time.  If you
 * pause the scrubber for a prolonged period of time you can use
 * the "read_all_memscrub" switch (see below) to catch up. In addition,
 * pause_memscrub is used internally by the post memory DR callbacks.
 * It is set for the small period of time during which the callbacks
 * are executing. This ensures "memscrub_lock" will be released,
 * allowing the callbacks to finish.
 *
 * The scrubber will read all memory if "read_all_memscrub" is set.
 * The normal span read will also occur during the wakeup.
 *
 * MEMSCRUB_MIN_PAGES (32MB) is the minimum amount of memory a system
 * must have before we'll start the scrubber.
 *
 * MEMSCRUB_DFL_SPAN_PAGES (32MB) is based on the guess that 0.025 sec
 * is a "good" amount of minimum time for the thread to run at a time.
 *
 * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess --
 * twice the frequency the hardware folk estimated would be necessary.
 *
 * MEMSCRUB_DFL_THREAD_PRI (MINCLSYSPRI) is based on the assumption
 * that the scurbber should get its fair share of time (since it
 * is short).  At a priority of 0 the scrubber will be starved.
 */

#include <sys/systm.h>          /* timeout, types, t_lock */
#include <sys/cmn_err.h>
#include <sys/sysmacros.h>      /* MIN */
#include <sys/memlist.h>        /* memlist */
#include <sys/mem_config.h>     /* memory add/delete */
#include <sys/kmem.h>           /* KMEM_NOSLEEP */
#include <sys/cpuvar.h>         /* ncpus_online */
#include <sys/debug.h>          /* ASSERTs */
#include <sys/machsystm.h>      /* lddphys */
#include <sys/cpu_module.h>     /* vtag_flushpage */
#include <sys/kstat.h>
#include <sys/atomic.h>         /* atomic_add_32 */

#include <vm/hat.h>
#include <vm/seg_kmem.h>
#include <vm/hat_sfmmu.h>       /* XXX FIXME - delete */

#include <sys/time.h>
#include <sys/callb.h>          /* CPR callback */
#include <sys/ontrap.h>

/*
 * Should really have paddr_t defined, but it is broken.  Use
 * ms_paddr_t in the meantime to make the code cleaner
 */
typedef uint64_t ms_paddr_t;

/*
 * Global Routines:
 */
int memscrub_add_span(pfn_t pfn, pgcnt_t pages);
int memscrub_delete_span(pfn_t pfn, pgcnt_t pages);
int memscrub_init(void);
void memscrub_induced_error(void);

/*
 * Global Data:
 */

/*
 * scrub if we have at least this many pages
 */
#define MEMSCRUB_MIN_PAGES (32 * 1024 * 1024 / PAGESIZE)

/*
 * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC
 */
#define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60)  /* 12 hours */

/*
 * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration
 */
#define MEMSCRUB_DFL_SPAN_PAGES ((32 * 1024 * 1024) / PAGESIZE)

/*
 * almost anything is higher priority than scrubbing
 */
#define MEMSCRUB_DFL_THREAD_PRI MINCLSYSPRI

/*
 * size used when scanning memory
 */
#define MEMSCRUB_BLOCK_SIZE             256
#define MEMSCRUB_BLOCK_SIZE_SHIFT       8       /* log2(MEMSCRUB_BLOCK_SIZE) */
#define MEMSCRUB_BLOCKS_PER_PAGE        (PAGESIZE >> MEMSCRUB_BLOCK_SIZE_SHIFT)

#define MEMSCRUB_BPP4M          MMU_PAGESIZE4M >> MEMSCRUB_BLOCK_SIZE_SHIFT
#define MEMSCRUB_BPP512K        MMU_PAGESIZE512K >> MEMSCRUB_BLOCK_SIZE_SHIFT
#define MEMSCRUB_BPP64K         MMU_PAGESIZE64K >> MEMSCRUB_BLOCK_SIZE_SHIFT
#define MEMSCRUB_BPP            MMU_PAGESIZE >> MEMSCRUB_BLOCK_SIZE_SHIFT

/*
 * This message indicates that we have exceeded the limitations of
 * the memscrubber. See the comments above regarding what would
 * cause the sleep time to become zero. In DEBUG mode, this message
 * is logged on the console and in the messages file. In non-DEBUG
 * mode, it is only logged in the messages file.
 */
#ifdef DEBUG
#define MEMSCRUB_OVERRIDE_MSG   "Memory scrubber sleep time is zero " \
        "seconds, consuming entire CPU."
#else
#define MEMSCRUB_OVERRIDE_MSG   "!Memory scrubber sleep time is zero " \
        "seconds, consuming entire CPU."
#endif /* DEBUG */

/*
 * we can patch these defaults in /etc/system if necessary
 */
uint_t disable_memscrub = 0;
uint_t pause_memscrub = 0;
uint_t read_all_memscrub = 0;
uint_t memscrub_verbose = 0;
uint_t memscrub_all_idle = 0;
uint_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES;
uint_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC;
uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI;
uint_t memscrub_delay_start_sec = 5 * 60;
uint_t memscrub_override_ticks = 1;

/*
 * Static Routines
 */
static void memscrubber(void);
static void memscrub_cleanup(void);
static int memscrub_add_span_gen(pfn_t, pgcnt_t, struct memlist **, uint_t *);
static int memscrub_verify_span(ms_paddr_t *addrp, pgcnt_t *pagesp);
static void memscrub_scan(uint_t blks, ms_paddr_t src);

/*
 * Static Data
 */

static struct memlist *memscrub_memlist;
static uint_t memscrub_phys_pages;

static kcondvar_t memscrub_cv;
static kmutex_t memscrub_lock;
/*
 * memscrub_lock protects memscrub_memlist, interval_ticks, cprinfo, ...
 */
static void memscrub_init_mem_config(void);
static void memscrub_uninit_mem_config(void);

/*
 * Linked list of memscrub aware spans having retired pages.
 * Currently enabled only on sun4u USIII-based platforms.
 */
typedef struct memscrub_page_retire_span {
        ms_paddr_t                              address;
        struct memscrub_page_retire_span        *next;
} memscrub_page_retire_span_t;

static memscrub_page_retire_span_t *memscrub_page_retire_span_list = NULL;

static void memscrub_page_retire_span_add(ms_paddr_t);
static void memscrub_page_retire_span_delete(ms_paddr_t);
static int memscrub_page_retire_span_search(ms_paddr_t);
static void memscrub_page_retire_span_list_update(void);

/*
 * add_to_page_retire_list: Set by cpu_async_log_err() routine
 * by calling memscrub_induced_error() when CE/UE occurs on a retired
 * page due to memscrub reading.  Cleared by memscrub after updating
 * global page retire span list.  Piggybacking on protection of
 * memscrub_lock, which is held during set and clear.
 * Note: When cpu_async_log_err() calls memscrub_induced_error(), it is running
 * on softint context, which gets fired on a cpu memscrub thread currently
 * running.  Memscrub thread has affinity set during memscrub_read(), hence
 * migration to new cpu not expected.
 */
static int add_to_page_retire_list = 0;

/*
 * Keep track of some interesting statistics
 */
static struct memscrub_kstats {
        kstat_named_t   done_early;     /* ahead of schedule */
        kstat_named_t   early_sec;      /* by cumulative num secs */
        kstat_named_t   done_late;      /* behind schedule */
        kstat_named_t   late_sec;       /* by cumulative num secs */
        kstat_named_t   interval_ticks; /* num ticks between intervals */
        kstat_named_t   force_run;      /* forced to run, non-timeout */
        kstat_named_t   errors_found;   /* num errors found by memscrub */
} memscrub_counts = {
        { "done_early",         KSTAT_DATA_UINT32 },
        { "early_sec",          KSTAT_DATA_UINT32 },
        { "done_late",          KSTAT_DATA_UINT32 },
        { "late_sec",           KSTAT_DATA_UINT32 },
        { "interval_ticks",     KSTAT_DATA_UINT32 },
        { "force_run",          KSTAT_DATA_UINT32 },
        { "errors_found",       KSTAT_DATA_UINT32 },
};

#define MEMSCRUB_STAT_INC(stat) memscrub_counts.stat.value.ui32++
#define MEMSCRUB_STAT_SET(stat, val) memscrub_counts.stat.value.ui32 = (val)
#define MEMSCRUB_STAT_NINC(stat, val) memscrub_counts.stat.value.ui32 += (val)

static struct kstat *memscrub_ksp = (struct kstat *)NULL;

static timeout_id_t memscrub_tid = 0;   /* keep track of timeout id */

/*
 * create memscrub_memlist from phys_install list
 * initialize locks, set memscrub_phys_pages.
 */
int
memscrub_init(void)
{
        struct memlist *src;

        /*
         * only startup the scrubber if we have a minimum
         * number of pages
         */
        if (physinstalled >= MEMSCRUB_MIN_PAGES) {

                /*
                 * initialize locks
                 */
                mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL);
                cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL);

                /*
                 * copy phys_install to memscrub_memlist
                 */
                for (src = phys_install; src; src = src->ml_next) {
                        if (memscrub_add_span(
                            (pfn_t)(src->ml_address >> PAGESHIFT),
                            (pgcnt_t)(src->ml_size >> PAGESHIFT))) {
                                memscrub_cleanup();
                                return (-1);
                        }
                }

                /*
                 * initialize kstats
                 */
                memscrub_ksp = kstat_create("unix", 0, "memscrub_kstat",
                    "misc", KSTAT_TYPE_NAMED,
                    sizeof (memscrub_counts) / sizeof (kstat_named_t),
                    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE);

                if (memscrub_ksp) {
                        memscrub_ksp->ks_data = (void *)&memscrub_counts;
                        kstat_install(memscrub_ksp);
                } else {
                        cmn_err(CE_NOTE, "Memscrubber cannot create kstats\n");
                }

                /*
                 * create memscrubber thread
                 */
                (void) thread_create(NULL, 0, (void (*)())memscrubber,
                    NULL, 0, &p0, TS_RUN, memscrub_thread_pri);

                /*
                 * We don't want call backs changing the list
                 * if there is no thread running. We do not
                 * attempt to deal with stopping/starting scrubbing
                 * on memory size changes.
                 */
                memscrub_init_mem_config();
        }

        return (0);
}

static void
memscrub_cleanup(void)
{
        memscrub_uninit_mem_config();
        while (memscrub_memlist) {
                (void) memscrub_delete_span(
                    (pfn_t)(memscrub_memlist->ml_address >> PAGESHIFT),
                    (pgcnt_t)(memscrub_memlist->ml_size >> PAGESHIFT));
        }
        if (memscrub_ksp)
                kstat_delete(memscrub_ksp);
        cv_destroy(&memscrub_cv);
        mutex_destroy(&memscrub_lock);
}

#ifdef MEMSCRUB_DEBUG
static void
memscrub_printmemlist(char *title, struct memlist *listp)
{
        struct memlist *list;

        cmn_err(CE_CONT, "%s:\n", title);

        for (list = listp; list; list = list->ml_next) {
                cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n",
                    list->ml_address, list->ml_size);
        }
}
#endif /* MEMSCRUB_DEBUG */

/* ARGSUSED */
static void
memscrub_wakeup(void *c)
{
        /*
         * grab mutex to guarantee that our wakeup call
         * arrives after we go to sleep -- so we can't sleep forever.
         */
        mutex_enter(&memscrub_lock);
        cv_signal(&memscrub_cv);
        mutex_exit(&memscrub_lock);
}

/*
 * provide an interface external to the memscrubber
 * which will force the memscrub thread to run vs.
 * waiting for the timeout, if one is set
 */
void
memscrub_run(void)
{
        MEMSCRUB_STAT_INC(force_run);
        if (memscrub_tid) {
                (void) untimeout(memscrub_tid);
                memscrub_wakeup((void *)NULL);
        }
}

/*
 * this calculation doesn't account for the time
 * that the actual scan consumes -- so we'd fall
 * slightly behind schedule with this interval.
 * It's very small.
 */

static uint_t
compute_interval_ticks(void)
{
        /*
         * We use msp_safe mpp_safe below to insure somebody
         * doesn't set memscrub_span_pages or memscrub_phys_pages
         * to 0 on us.
         */
        static uint_t msp_safe, mpp_safe;
        static uint_t interval_ticks, period_ticks;
        msp_safe = memscrub_span_pages;
        mpp_safe = memscrub_phys_pages;

        period_ticks = memscrub_period_sec * hz;
        interval_ticks = period_ticks;

        ASSERT(mutex_owned(&memscrub_lock));

        if ((msp_safe != 0) && (mpp_safe != 0)) {
                if (memscrub_phys_pages <= msp_safe) {
                        interval_ticks = period_ticks;
                } else {
                        interval_ticks = (period_ticks /
                            (mpp_safe / msp_safe));
                }
        }
        return (interval_ticks);
}

void
memscrubber(void)
{
        ms_paddr_t address, addr;
        time_t deadline;
        pgcnt_t pages;
        uint_t reached_end = 1;
        uint_t paused_message = 0;
        uint_t interval_ticks = 0;
        uint_t sleep_warn_printed = 0;
        callb_cpr_t cprinfo;

        /*
         * notify CPR of our existence
         */
        CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub");

        mutex_enter(&memscrub_lock);

        if (memscrub_memlist == NULL) {
                cmn_err(CE_WARN, "memscrub_memlist not initialized.");
                goto memscrub_exit;
        }

        address = memscrub_memlist->ml_address;

        deadline = gethrestime_sec() + memscrub_delay_start_sec;

        for (;;) {
                if (disable_memscrub)
                        break;

                /*
                 * compute interval_ticks
                 */
                interval_ticks = compute_interval_ticks();

                /*
                 * If the calculated sleep time is zero, and pause_memscrub
                 * has been set, make sure we sleep so that another thread
                 * can acquire memscrub_lock.
                 */
                if (interval_ticks == 0 && pause_memscrub) {
                        interval_ticks = hz;
                }

                /*
                 * And as a fail safe, under normal non-paused operation, do
                 * not allow the sleep time to be zero.
                 */
                if (interval_ticks == 0) {
                        interval_ticks = memscrub_override_ticks;
                        if (!sleep_warn_printed) {
                                cmn_err(CE_NOTE, MEMSCRUB_OVERRIDE_MSG);
                                sleep_warn_printed = 1;
                        }
                }

                MEMSCRUB_STAT_SET(interval_ticks, interval_ticks);

                /*
                 * Did we just reach the end of memory? If we are at the
                 * end of memory, delay end of memory processing until
                 * pause_memscrub is not set.
                 */
                if (reached_end && !pause_memscrub) {
                        time_t now = gethrestime_sec();

                        if (now >= deadline) {
                                MEMSCRUB_STAT_INC(done_late);
                                MEMSCRUB_STAT_NINC(late_sec, now - deadline);
                                /*
                                 * past deadline, start right away
                                 */
                                interval_ticks = 0;

                                deadline = now + memscrub_period_sec;
                        } else {
                                /*
                                 * we finished ahead of schedule.
                                 * wait till previous deadline before re-start.
                                 */
                                interval_ticks = (deadline - now) * hz;
                                MEMSCRUB_STAT_INC(done_early);
                                MEMSCRUB_STAT_NINC(early_sec, deadline - now);
                                deadline += memscrub_period_sec;
                        }
                        reached_end = 0;
                        sleep_warn_printed = 0;
                }

                if (interval_ticks != 0) {
                        /*
                         * it is safe from our standpoint for CPR to
                         * suspend the system
                         */
                        CALLB_CPR_SAFE_BEGIN(&cprinfo);

                        /*
                         * hit the snooze bar
                         */
                        memscrub_tid = timeout(memscrub_wakeup, NULL,
                            interval_ticks);

                        /*
                         * go to sleep
                         */
                        cv_wait(&memscrub_cv, &memscrub_lock);

                        /*
                         * at this point, no timeout should be set
                         */
                        memscrub_tid = 0;

                        /*
                         * we need to goto work and will be modifying
                         * our internal state and mapping/unmapping
                         * TTEs
                         */
                        CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock);
                }


                if (memscrub_phys_pages == 0) {
                        cmn_err(CE_WARN, "Memory scrubber has 0 pages to read");
                        goto memscrub_exit;
                }

                if (!pause_memscrub) {
                        if (paused_message) {
                                paused_message = 0;
                                if (memscrub_verbose)
                                        cmn_err(CE_NOTE, "Memory scrubber "
                                            "resuming");
                        }

                        if (read_all_memscrub) {
                                if (memscrub_verbose)
                                        cmn_err(CE_NOTE, "Memory scrubber "
                                            "reading all memory per request");

                                addr = memscrub_memlist->ml_address;
                                reached_end = 0;
                                while (!reached_end) {
                                        if (disable_memscrub)
                                                break;
                                        pages = memscrub_phys_pages;
                                        reached_end = memscrub_verify_span(
                                            &addr, &pages);
                                        memscrub_scan(pages *
                                            MEMSCRUB_BLOCKS_PER_PAGE, addr);
                                        addr += ((uint64_t)pages * PAGESIZE);
                                }
                                read_all_memscrub = 0;
                        }

                        /*
                         * read 1 span
                         */
                        pages = memscrub_span_pages;

                        if (disable_memscrub)
                                break;

                        /*
                         * determine physical address range
                         */
                        reached_end = memscrub_verify_span(&address,
                            &pages);

                        memscrub_scan(pages * MEMSCRUB_BLOCKS_PER_PAGE,
                            address);

                        address += ((uint64_t)pages * PAGESIZE);
                }

                if (pause_memscrub && !paused_message) {
                        paused_message = 1;
                        if (memscrub_verbose)
                                cmn_err(CE_NOTE, "Memory scrubber paused");
                }
        }

memscrub_exit:
        cmn_err(CE_NOTE, "Memory scrubber exiting");
        CALLB_CPR_EXIT(&cprinfo);
        memscrub_cleanup();
        thread_exit();
        /* NOTREACHED */
}

/*
 * condition address and size
 * such that they span legal physical addresses.
 *
 * when appropriate, address will be rounded up to start of next
 * struct memlist, and pages will be rounded down to the end of the
 * memlist size.
 *
 * returns 1 if reached end of list, else returns 0.
 */
static int
memscrub_verify_span(ms_paddr_t *addrp, pgcnt_t *pagesp)
{
        struct memlist *mlp;
        ms_paddr_t address = *addrp;
        uint64_t bytes = (uint64_t)*pagesp * PAGESIZE;
        uint64_t bytes_remaining;
        int reached_end = 0;

        ASSERT(mutex_owned(&memscrub_lock));

        /*
         * find memlist struct that contains addrp
         * assumes memlist is sorted by ascending address.
         */
        for (mlp = memscrub_memlist; mlp != NULL; mlp = mlp->ml_next) {
                /*
                 * if before this chunk, round up to beginning
                 */
                if (address < mlp->ml_address) {
                        address = mlp->ml_address;
                        break;
                }
                /*
                 * if before end of chunk, then we found it
                 */
                if (address < (mlp->ml_address + mlp->ml_size))
                        break;

                /* else go to next struct memlist */
        }
        /*
         * if we hit end of list, start at beginning
         */
        if (mlp == NULL) {
                mlp = memscrub_memlist;
                address = mlp->ml_address;
        }

        /*
         * now we have legal address, and its mlp, condition bytes
         */
        bytes_remaining = (mlp->ml_address + mlp->ml_size) - address;

        if (bytes > bytes_remaining)
                bytes = bytes_remaining;

        /*
         * will this span take us to end of list?
         */
        if ((mlp->ml_next == NULL) &&
            ((mlp->ml_address + mlp->ml_size) == (address + bytes)))
                reached_end = 1;

        /* return values */
        *addrp = address;
        *pagesp = bytes / PAGESIZE;

        return (reached_end);
}

/*
 * add a span to the memscrub list
 * add to memscrub_phys_pages
 */
int
memscrub_add_span(pfn_t pfn, pgcnt_t pages)
{
#ifdef MEMSCRUB_DEBUG
        ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT;
        uint64_t bytes = (uint64_t)pages << PAGESHIFT;
#endif /* MEMSCRUB_DEBUG */

        int retval;

        mutex_enter(&memscrub_lock);

#ifdef MEMSCRUB_DEBUG
        memscrub_printmemlist("memscrub_memlist before", memscrub_memlist);
        cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
        cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx"
            " size: 0x%llx\n", address, bytes);
#endif /* MEMSCRUB_DEBUG */

        retval = memscrub_add_span_gen(pfn, pages, &memscrub_memlist,
            &memscrub_phys_pages);

#ifdef MEMSCRUB_DEBUG
        memscrub_printmemlist("memscrub_memlist after", memscrub_memlist);
        cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
#endif /* MEMSCRUB_DEBUG */

        mutex_exit(&memscrub_lock);

        return (retval);
}

static int
memscrub_add_span_gen(
        pfn_t pfn,
        pgcnt_t pages,
        struct memlist **list,
        uint_t *npgs)
{
        ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT;
        uint64_t bytes = (uint64_t)pages << PAGESHIFT;
        struct memlist *dst;
        struct memlist *prev, *next;
        int retval = 0;

        /*
         * allocate a new struct memlist
         */

        dst = (struct memlist *)
            kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);

        if (dst == NULL) {
                retval = -1;
                goto add_done;
        }

        dst->ml_address = address;
        dst->ml_size = bytes;

        /*
         * first insert
         */
        if (*list == NULL) {
                dst->ml_prev = NULL;
                dst->ml_next = NULL;
                *list = dst;

                goto add_done;
        }

        /*
         * insert into sorted list
         */
        for (prev = NULL, next = *list;
            next != NULL;
            prev = next, next = next->ml_next) {
                if (address > (next->ml_address + next->ml_size))
                        continue;

                /*
                 * else insert here
                 */

                /*
                 * prepend to next
                 */
                if ((address + bytes) == next->ml_address) {
                        kmem_free(dst, sizeof (struct memlist));

                        next->ml_address = address;
                        next->ml_size += bytes;

                        goto add_done;
                }

                /*
                 * append to next
                 */
                if (address == (next->ml_address + next->ml_size)) {
                        kmem_free(dst, sizeof (struct memlist));

                        if (next->ml_next) {
                                /*
                                 * don't overlap with next->ml_next
                                 */
                                if ((address + bytes) >
                                    next->ml_next->ml_address) {
                                        retval = -1;
                                        goto add_done;
                                }
                                /*
                                 * concatenate next and next->ml_next
                                 */
                                if ((address + bytes) ==
                                    next->ml_next->ml_address) {
                                        struct memlist *mlp = next->ml_next;

                                        if (next == *list)
                                                *list = next->ml_next;

                                        mlp->ml_address = next->ml_address;
                                        mlp->ml_size += next->ml_size;
                                        mlp->ml_size += bytes;

                                        if (next->ml_prev)
                                                next->ml_prev->ml_next = mlp;
                                        mlp->ml_prev = next->ml_prev;

                                        kmem_free(next,
                                            sizeof (struct memlist));
                                        goto add_done;
                                }
                        }

                        next->ml_size += bytes;

                        goto add_done;
                }

                /* don't overlap with next */
                if ((address + bytes) > next->ml_address) {
                        retval = -1;
                        kmem_free(dst, sizeof (struct memlist));
                        goto add_done;
                }

                /*
                 * insert before next
                 */
                dst->ml_prev = prev;
                dst->ml_next = next;
                next->ml_prev = dst;
                if (prev == NULL) {
                        *list = dst;
                } else {
                        prev->ml_next = dst;
                }
                goto add_done;
        }       /* end for */

        /*
         * end of list, prev is valid and next is NULL
         */
        prev->ml_next = dst;
        dst->ml_prev = prev;
        dst->ml_next = NULL;

add_done:

        if (retval != -1)
                *npgs += pages;

        return (retval);
}

/*
 * delete a span from the memscrub list
 * subtract from memscrub_phys_pages
 */
int
memscrub_delete_span(pfn_t pfn, pgcnt_t pages)
{
        ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT;
        uint64_t bytes = (uint64_t)pages << PAGESHIFT;
        struct memlist *dst, *next;
        int retval = 0;

        mutex_enter(&memscrub_lock);

#ifdef MEMSCRUB_DEBUG
        memscrub_printmemlist("memscrub_memlist Before", memscrub_memlist);
        cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
        cmn_err(CE_CONT, "memscrub_delete_span: 0x%llx 0x%llx\n",
            address, bytes);
#endif /* MEMSCRUB_DEBUG */

        /*
         * find struct memlist containing page
         */
        for (next = memscrub_memlist; next != NULL; next = next->ml_next) {
                if ((address >= next->ml_address) &&
                    (address < next->ml_address + next->ml_size))
                        break;
        }

        /*
         * if start address not in list
         */
        if (next == NULL) {
                retval = -1;
                goto delete_done;
        }

        /*
         * error if size goes off end of this struct memlist
         */
        if (address + bytes > next->ml_address + next->ml_size) {
                retval = -1;
                goto delete_done;
        }

        /*
         * pages at beginning of struct memlist
         */
        if (address == next->ml_address) {
                /*
                 * if start & size match, delete from list
                 */
                if (bytes == next->ml_size) {
                        if (next == memscrub_memlist)
                                memscrub_memlist = next->ml_next;
                        if (next->ml_prev != NULL)
                                next->ml_prev->ml_next = next->ml_next;
                        if (next->ml_next != NULL)
                                next->ml_next->ml_prev = next->ml_prev;

                        kmem_free(next, sizeof (struct memlist));
                } else {
                /*
                 * increment start address by bytes
                 */
                        next->ml_address += bytes;
                        next->ml_size -= bytes;
                }
                goto delete_done;
        }

        /*
         * pages at end of struct memlist
         */
        if (address + bytes == next->ml_address + next->ml_size) {
                /*
                 * decrement size by bytes
                 */
                next->ml_size -= bytes;
                goto delete_done;
        }

        /*
         * delete a span in the middle of the struct memlist
         */
        {
                /*
                 * create a new struct memlist
                 */
                dst = (struct memlist *)
                    kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);

                if (dst == NULL) {
                        retval = -1;
                        goto delete_done;
                }

                /*
                 * existing struct memlist gets address
                 * and size up to pfn
                 */
                dst->ml_address = address + bytes;
                dst->ml_size =
                    (next->ml_address + next->ml_size) - dst->ml_address;
                next->ml_size = address - next->ml_address;

                /*
                 * new struct memlist gets address starting
                 * after pfn, until end
                 */

                /*
                 * link in new memlist after old
                 */
                dst->ml_next = next->ml_next;
                dst->ml_prev = next;

                if (next->ml_next != NULL)
                        next->ml_next->ml_prev = dst;
                next->ml_next = dst;
        }

delete_done:
        if (retval != -1) {
                memscrub_phys_pages -= pages;
                if (memscrub_phys_pages == 0)
                        disable_memscrub = 1;
        }

#ifdef MEMSCRUB_DEBUG
        memscrub_printmemlist("memscrub_memlist After", memscrub_memlist);
        cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
#endif /* MEMSCRUB_DEBUG */

        mutex_exit(&memscrub_lock);
        return (retval);
}

static void
memscrub_scan(uint_t blks, ms_paddr_t src)
{
        uint_t          psz, bpp, pgsread;
        pfn_t           pfn;
        ms_paddr_t      pa;
        caddr_t         va;
        on_trap_data_t  otd;
        int             scan_mmu_pagesize = 0;
        int             retired_pages = 0;

        extern void memscrub_read(caddr_t src, uint_t blks);

        ASSERT(mutex_owned(&memscrub_lock));

        pgsread = 0;
        pa = src;

        if (memscrub_page_retire_span_list != NULL) {
                if (memscrub_page_retire_span_search(src)) {
                        /* retired pages in current span */
                        scan_mmu_pagesize = 1;
                }
        }

#ifdef MEMSCRUB_DEBUG
        cmn_err(CE_NOTE, "scan_mmu_pagesize = %d\n" scan_mmu_pagesize);
#endif /* MEMSCRUB_DEBUG */

        while (blks != 0) {
                /* Ensure the PA is properly aligned */
                if (((pa & MMU_PAGEMASK4M) == pa) &&
                    (blks >= MEMSCRUB_BPP4M)) {
                        psz = MMU_PAGESIZE4M;
                        bpp = MEMSCRUB_BPP4M;
                } else if (((pa & MMU_PAGEMASK512K) == pa) &&
                    (blks >= MEMSCRUB_BPP512K)) {
                        psz = MMU_PAGESIZE512K;
                        bpp = MEMSCRUB_BPP512K;
                } else if (((pa & MMU_PAGEMASK64K) == pa) &&
                    (blks >= MEMSCRUB_BPP64K)) {
                        psz = MMU_PAGESIZE64K;
                        bpp = MEMSCRUB_BPP64K;
                } else if ((pa & MMU_PAGEMASK) == pa) {
                        psz = MMU_PAGESIZE;
                        bpp = MEMSCRUB_BPP;
                } else {
                        if (memscrub_verbose) {
                                cmn_err(CE_NOTE, "Memory scrubber ignoring "
                                    "non-page aligned block starting at 0x%"
                                    PRIx64, src);
                        }
                        return;
                }
                if (blks < bpp) bpp = blks;

#ifdef MEMSCRUB_DEBUG
                cmn_err(CE_NOTE, "Going to run psz=%x, "
                    "bpp=%x pa=%llx\n", psz, bpp, pa);
#endif /* MEMSCRUB_DEBUG */

                /*
                 * MEMSCRUBBASE is a 4MB aligned page in the
                 * kernel so that we can quickly map the PA
                 * to a VA for the block loads performed in
                 * memscrub_read.
                 */
                pfn = mmu_btop(pa);
                va = (caddr_t)MEMSCRUBBASE;
                hat_devload(kas.a_hat, va, psz, pfn, PROT_READ,
                    HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);

                /*
                 * Can't allow the memscrubber to migrate across CPUs as
                 * we need to know whether CEEN is enabled for the current
                 * CPU to enable us to scrub the memory. Don't use
                 * kpreempt_disable as the time we take to scan a span (even
                 * without cpu_check_ce having to manually cpu_check_block)
                 * is too long to hold a higher priority thread (eg, RT)
                 * off cpu.
                 */
                thread_affinity_set(curthread, CPU_CURRENT);

                /*
                 * Protect read scrub from async faults.  For now, we simply
                 * maintain a count of such faults caught.
                 */

                if (!on_trap(&otd, OT_DATA_EC) && !scan_mmu_pagesize) {
                        memscrub_read(va, bpp);
                        /*
                         * Check if CEs require logging
                         */
                        cpu_check_ce(SCRUBBER_CEEN_CHECK,
                            (uint64_t)pa, va, psz);
                        no_trap();
                        thread_affinity_clear(curthread);
                } else {
                        no_trap();
                        thread_affinity_clear(curthread);

                        /*
                         * Got an async error..
                         * Try rescanning it at MMU_PAGESIZE
                         * granularity if we were trying to
                         * read at a larger page size.
                         * This is to ensure we continue to
                         * scan the rest of the span.
                         * OR scanning MMU_PAGESIZE granularity to avoid
                         * reading retired pages memory when scan_mmu_pagesize
                         * is set.
                         */
                        if (psz > MMU_PAGESIZE || scan_mmu_pagesize) {
                                caddr_t vaddr = va;
                                ms_paddr_t paddr = pa;
                                int tmp = 0;
                                for (; tmp < bpp; tmp += MEMSCRUB_BPP) {
                                        /* Don't scrub retired pages */
                                        if (page_retire_check(paddr, NULL)
                                            == 0) {
                                                vaddr += MMU_PAGESIZE;
                                                paddr += MMU_PAGESIZE;
                                                retired_pages++;
                                                continue;
                                        }
                                        thread_affinity_set(curthread,
                                            CPU_CURRENT);
                                        if (!on_trap(&otd, OT_DATA_EC)) {
                                                memscrub_read(vaddr,
                                                    MEMSCRUB_BPP);
                                                cpu_check_ce(
                                                    SCRUBBER_CEEN_CHECK,
                                                    (uint64_t)paddr, vaddr,
                                                    MMU_PAGESIZE);
                                                no_trap();
                                        } else {
                                                no_trap();
                                                MEMSCRUB_STAT_INC(errors_found);
                                        }
                                        thread_affinity_clear(curthread);
                                        vaddr += MMU_PAGESIZE;
                                        paddr += MMU_PAGESIZE;
                                }
                        }
                }
                hat_unload(kas.a_hat, va, psz, HAT_UNLOAD_UNLOCK);

                blks -= bpp;
                pa += psz;
                pgsread++;
        }

        /*
         * If just finished scrubbing MMU_PAGESIZE at a time, but no retired
         * pages found so delete span from global list.
         */
        if (scan_mmu_pagesize && retired_pages == 0)
                memscrub_page_retire_span_delete(src);

        /*
         * Encountered CE/UE on a retired page during memscrub read of current
         * span.  Adding span to global list to enable avoid reading further.
         */
        if (add_to_page_retire_list) {
                if (!memscrub_page_retire_span_search(src))
                        memscrub_page_retire_span_add(src);
                add_to_page_retire_list = 0;
        }

        if (memscrub_verbose) {
                cmn_err(CE_NOTE, "Memory scrubber read 0x%x pages starting "
                    "at 0x%" PRIx64, pgsread, src);
        }
}

/*
 * Called by cpu_async_log_err() when memscrub read causes
 * CE/UE on a retired page.
 */
void
memscrub_induced_error(void)
{
        add_to_page_retire_list = 1;
}

/*
 * Called by page_retire() when toxic pages cannot be retired
 * immediately and are scheduled for retire.  Memscrubber stops
 * scrubbing them to avoid further CE/UEs.
 */
void
memscrub_notify(ms_paddr_t pa)
{
        mutex_enter(&memscrub_lock);
        if (!memscrub_page_retire_span_search(pa))
                memscrub_page_retire_span_add(pa);
        mutex_exit(&memscrub_lock);
}

/*
 * Called by memscrub_scan() and memscrub_notify().
 * pa: physical address of span with CE/UE, add to global list.
 */
static void
memscrub_page_retire_span_add(ms_paddr_t pa)
{
        memscrub_page_retire_span_t *new_span;

        new_span = (memscrub_page_retire_span_t *)
            kmem_zalloc(sizeof (memscrub_page_retire_span_t), KM_NOSLEEP);

        if (new_span == NULL) {
#ifdef MEMSCRUB_DEBUG
                cmn_err(CE_NOTE, "failed to allocate new span - span with"
                    " retired page/s not tracked.\n");
#endif /* MEMSCRUB_DEBUG */
                return;
        }

        new_span->address = pa;
        new_span->next = memscrub_page_retire_span_list;
        memscrub_page_retire_span_list = new_span;
}

/*
 * Called by memscrub_scan().
 * pa: physical address of span to be removed from global list.
 */
static void
memscrub_page_retire_span_delete(ms_paddr_t pa)
{
        memscrub_page_retire_span_t *prev_span, *next_span;

        prev_span = memscrub_page_retire_span_list;
        next_span = memscrub_page_retire_span_list->next;

        if (pa == prev_span->address) {
                memscrub_page_retire_span_list = next_span;
                kmem_free(prev_span, sizeof (memscrub_page_retire_span_t));
                return;
        }

        while (next_span) {
                if (pa == next_span->address) {
                        prev_span->next = next_span->next;
                        kmem_free(next_span,
                            sizeof (memscrub_page_retire_span_t));
                        return;
                }
                prev_span = next_span;
                next_span = next_span->next;
        }
}

/*
 * Called by memscrub_scan() and memscrub_notify().
 * pa: physical address of span to be searched in global list.
 */
static int
memscrub_page_retire_span_search(ms_paddr_t pa)
{
        memscrub_page_retire_span_t *next_span = memscrub_page_retire_span_list;

        while (next_span) {
                if (pa == next_span->address)
                        return (1);
                next_span = next_span->next;
        }
        return (0);
}

/*
 * Called from new_memscrub() as a result of memory delete.
 * Using page_numtopp_nolock() to determine if we have valid PA.
 */
static void
memscrub_page_retire_span_list_update(void)
{
        memscrub_page_retire_span_t *prev, *cur, *next;

        if (memscrub_page_retire_span_list == NULL)
                return;

        prev = cur = memscrub_page_retire_span_list;
        next = cur->next;

        while (cur) {
                if (page_numtopp_nolock(mmu_btop(cur->address)) == NULL) {
                        if (cur == memscrub_page_retire_span_list) {
                                memscrub_page_retire_span_list = next;
                                kmem_free(cur,
                                    sizeof (memscrub_page_retire_span_t));
                                prev = cur = memscrub_page_retire_span_list;
                        } else {
                                prev->next = cur->next;
                                kmem_free(cur,
                                    sizeof (memscrub_page_retire_span_t));
                                cur = next;
                        }
                } else {
                        prev = cur;
                        cur = next;
                }
                if (cur != NULL)
                        next = cur->next;
        }
}

/*
 * The memory add/delete callback mechanism does not pass in the
 * page ranges. The phys_install list has been updated though, so
 * create a new scrub list from it.
 */

static int
new_memscrub(int update_page_retire_list)
{
        struct memlist *src, *list, *old_list;
        uint_t npgs;

        /*
         * copy phys_install to memscrub_memlist
         */
        list = NULL;
        npgs = 0;
        memlist_read_lock();
        for (src = phys_install; src; src = src->ml_next) {
                if (memscrub_add_span_gen((pfn_t)(src->ml_address >> PAGESHIFT),
                    (pgcnt_t)(src->ml_size >> PAGESHIFT), &list, &npgs)) {
                        memlist_read_unlock();
                        while (list) {
                                struct memlist *el;

                                el = list;
                                list = list->ml_next;
                                kmem_free(el, sizeof (struct memlist));
                        }
                        return (-1);
                }
        }
        memlist_read_unlock();

        mutex_enter(&memscrub_lock);
        memscrub_phys_pages = npgs;
        old_list = memscrub_memlist;
        memscrub_memlist = list;

        if (update_page_retire_list)
                memscrub_page_retire_span_list_update();

        mutex_exit(&memscrub_lock);

        while (old_list) {
                struct memlist *el;

                el = old_list;
                old_list = old_list->ml_next;
                kmem_free(el, sizeof (struct memlist));
        }

        return (0);
}

/*ARGSUSED*/
static void
memscrub_mem_config_post_add(
        void *arg,
        pgcnt_t delta_pages)
{
        /*
         * We increment pause_memscrub before entering new_memscrub(). This
         * will force the memscrubber to sleep, allowing the DR callback
         * thread to acquire memscrub_lock in new_memscrub(). The use of
         * atomic_add_32() allows concurrent memory DR operations to use the
         * callbacks safely.
         */
        atomic_inc_32(&pause_memscrub);
        ASSERT(pause_memscrub != 0);

        /*
         * "Don't care" if we are not scrubbing new memory.
         */
        (void) new_memscrub(0);         /* retain page retire list */

        /* Restore the pause setting. */
        atomic_dec_32(&pause_memscrub);
}

/*ARGSUSED*/
static int
memscrub_mem_config_pre_del(
        void *arg,
        pgcnt_t delta_pages)
{
        /* Nothing to do. */
        return (0);
}

/*ARGSUSED*/
static void
memscrub_mem_config_post_del(
        void *arg,
        pgcnt_t delta_pages,
        int cancelled)
{
        /*
         * We increment pause_memscrub before entering new_memscrub(). This
         * will force the memscrubber to sleep, allowing the DR callback
         * thread to acquire memscrub_lock in new_memscrub(). The use of
         * atomic_add_32() allows concurrent memory DR operations to use the
         * callbacks safely.
         */
        atomic_inc_32(&pause_memscrub);
        ASSERT(pause_memscrub != 0);

        /*
         * Must stop scrubbing deleted memory as it may be disconnected.
         */
        if (new_memscrub(1)) {  /* update page retire list */
                disable_memscrub = 1;
        }

        /* Restore the pause setting. */
        atomic_dec_32(&pause_memscrub);
}

static kphysm_setup_vector_t memscrub_mem_config_vec = {
        KPHYSM_SETUP_VECTOR_VERSION,
        memscrub_mem_config_post_add,
        memscrub_mem_config_pre_del,
        memscrub_mem_config_post_del,
};

static void
memscrub_init_mem_config()
{
        int ret;

        ret = kphysm_setup_func_register(&memscrub_mem_config_vec,
            (void *)NULL);
        ASSERT(ret == 0);
}

static void
memscrub_uninit_mem_config()
{
        /* This call is OK if the register call was not done. */
        kphysm_setup_func_unregister(&memscrub_mem_config_vec, (void *)NULL);
}