/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * i86pc Memory Scrubbing
 *
 * On detection of a correctable memory ECC error, the i86pc hardware
 * returns the corrected data to the requester and may re-write it
 * to memory (DRAM or NVRAM). Machines which do not re-write this to
 * memory should add an NMI handler to correct and rewrite.
 *
 * Scrubbing thus reduces the likelyhood that multiple transient errors
 * will occur in the same memory word, making uncorrectable errors due
 * to transients less likely.
 *
 * Thus is born the desire that every memory location be periodically
 * accessed.
 *
 * This file implements a memory scrubbing thread.  This scrubber
 * guarantees that all of physical memory is accessed periodically
 * (memscrub_period_sec -- 12 hours).
 *
 * It attempts to do this as unobtrusively as possible.  The thread
 * schedules itself to wake up at an interval such that if it reads
 * memscrub_span_pages (4MB) on each wakeup, it will read all of physical
 * memory in in memscrub_period_sec (12 hours).
 *
 * The scrubber uses the REP LODS so it reads 4MB in 0.15 secs (on P5-200).
 * When it completes a span, if all the CPUs are idle, it reads another span.
 * Typically it soaks up idle time this way to reach its deadline early
 * -- and sleeps until the next period begins.
 *
 * Maximal Cost Estimate:  8GB @ xxMB/s = xxx seconds spent in 640 wakeups
 * that run for 0.15 seconds at intervals of 67 seconds.
 *
 * In practice, the scrubber finds enough idle time to finish in a few
 * minutes, and sleeps until its 12 hour deadline.
 *
 * The scrubber maintains a private copy of the phys_install memory list
 * to keep track of what memory should be scrubbed.
 *
 * The following parameters can be set via /etc/system
 *
 * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (4MB)
 * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours)
 * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (0)
 * memscrub_delay_start_sec = (10 seconds)
 * disable_memscrub = (0)
 *
 * the scrubber will exit (or never be started) if it finds the variable
 * "disable_memscrub" set.
 *
 * MEMSCRUB_DFL_SPAN_PAGES  is based on the guess that 0.15 sec
 * is a "good" amount of minimum time for the thread to run at a time.
 *
 * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess --
 * twice the frequency the hardware folk estimated would be necessary.
 *
 * MEMSCRUB_DFL_THREAD_PRI (0) is based on the assumption that nearly
 * any other use of the system should be higher priority than scrubbing.
 */

#include <sys/types.h>
#include <sys/systm.h>          /* timeout, types, t_lock */
#include <sys/cmn_err.h>
#include <sys/sysmacros.h>      /* MIN */
#include <sys/memlist.h>        /* memlist */
#include <sys/kmem.h>           /* KMEM_NOSLEEP */
#include <sys/cpuvar.h>         /* ncpus_online */
#include <sys/debug.h>          /* ASSERTs */
#include <sys/vmem.h>
#include <sys/mman.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/hat_i86.h>
#include <sys/callb.h>          /* CPR callback */

static caddr_t  memscrub_window;
static hat_mempte_t memscrub_pte;

/*
 * Global Data:
 */
/*
 * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC
 */
#define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60)  /* 12 hours */

/*
 * start only if at least MEMSCRUB_MIN_PAGES in system
 */
#define MEMSCRUB_MIN_PAGES      ((32 * 1024 * 1024) / PAGESIZE)

/*
 * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration
 */
#define MEMSCRUB_DFL_SPAN_PAGES ((4 * 1024 * 1024) / PAGESIZE)

/*
 * almost anything is higher priority than scrubbing
 */
#define MEMSCRUB_DFL_THREAD_PRI 0

/*
 * we can patch these defaults in /etc/system if necessary
 */
uint_t disable_memscrub = 0;
static uint_t disable_memscrub_quietly = 0;
pgcnt_t memscrub_min_pages = MEMSCRUB_MIN_PAGES;
pgcnt_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES;
time_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC;
uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI;
time_t memscrub_delay_start_sec = 10;

/*
 * Static Routines
 */
static void memscrubber(void);
static int system_is_idle(void);
static int memscrub_add_span(uint64_t, uint64_t);

/*
 * Static Data
 */
static struct memlist *memscrub_memlist;
static uint_t memscrub_phys_pages;

static kcondvar_t memscrub_cv;
static kmutex_t memscrub_lock;

/*
 * memscrub_lock protects memscrub_memlist
 */
uint_t memscrub_scans_done;

uint_t memscrub_done_early;
uint_t memscrub_early_sec;

uint_t memscrub_done_late;
time_t memscrub_late_sec;

/*
 * create memscrub_memlist from phys_install list
 * initialize locks, set memscrub_phys_pages.
 */
void
memscrub_init()
{
        struct memlist *src;

        if (physmem < memscrub_min_pages)
                return;

        if (!kpm_enable) {
                memscrub_window = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
                memscrub_pte = hat_mempte_setup(memscrub_window);
        }

        /*
         * copy phys_install to memscrub_memlist
         */
        for (src = phys_install; src; src = src->ml_next) {
                if (memscrub_add_span(src->ml_address, src->ml_size)) {
                        cmn_err(CE_WARN,
                            "Software memory scrubber failed to initialize\n");
                        return;
                }
        }

        mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL);
        cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL);

        /*
         * create memscrubber thread
         */
        (void) thread_create(NULL, 0, (void (*)())memscrubber, NULL, 0, &p0,
            TS_RUN, memscrub_thread_pri);
}

/*
 * Function to cause the software memscrubber to exit quietly if the
 * platform support has located a hardware scrubber and enabled it.
 */
void
memscrub_disable(void)
{
        disable_memscrub_quietly = 1;
}

#ifdef MEMSCRUB_DEBUG
static void
memscrub_printmemlist(char *title, struct memlist *listp)
{
        struct memlist *list;

        cmn_err(CE_CONT, "%s:\n", title);

        for (list = listp; list; list = list->next) {
                cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n",
                    list->address, list->size);
        }
}
#endif /* MEMSCRUB_DEBUG */

/* ARGSUSED */
static void
memscrub_wakeup(void *c)
{
        /*
         * grab mutex to guarantee that our wakeup call
         * arrives after we go to sleep -- so we can't sleep forever.
         */
        mutex_enter(&memscrub_lock);
        cv_signal(&memscrub_cv);
        mutex_exit(&memscrub_lock);
}

/*
 * this calculation doesn't account for the time that the actual scan
 * consumes -- so we'd fall slightly behind schedule with this
 * interval_sec.  but the idle loop optimization below usually makes us
 * come in way ahead of schedule.
 */
static int
compute_interval_sec()
{
        if (memscrub_phys_pages <= memscrub_span_pages)
                return (memscrub_period_sec);
        else
                return (memscrub_period_sec/
                    (memscrub_phys_pages/memscrub_span_pages));
}

static void
memscrubber()
{
        time_t deadline;
        uint64_t mlp_last_addr;
        uint64_t mlp_next_addr;
        int reached_end = 1;
        time_t interval_sec = 0;
        struct memlist *mlp;

        extern void scan_memory(caddr_t, size_t);
        callb_cpr_t cprinfo;

        /*
         * notify CPR of our existence
         */
        CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub");

        if (memscrub_memlist == NULL) {
                cmn_err(CE_WARN, "memscrub_memlist not initialized.");
                goto memscrub_exit;
        }

        mlp = memscrub_memlist;
        mlp_next_addr = mlp->ml_address;
        mlp_last_addr = mlp->ml_address + mlp->ml_size;

        deadline = gethrestime_sec() + memscrub_delay_start_sec;

        for (;;) {
                if (disable_memscrub || disable_memscrub_quietly)
                        break;

                mutex_enter(&memscrub_lock);

                /*
                 * did we just reach the end of memory?
                 */
                if (reached_end) {
                        time_t now = gethrestime_sec();

                        if (now >= deadline) {
                                memscrub_done_late++;
                                memscrub_late_sec += (now - deadline);
                                /*
                                 * past deadline, start right away
                                 */
                                interval_sec = 0;

                                deadline = now + memscrub_period_sec;
                        } else {
                                /*
                                 * we finished ahead of schedule.
                                 * wait till previous dealine before re-start.
                                 */
                                interval_sec = deadline - now;
                                memscrub_done_early++;
                                memscrub_early_sec += interval_sec;
                                deadline += memscrub_period_sec;
                        }
                } else {
                        interval_sec = compute_interval_sec();
                }

                /*
                 * it is safe from our standpoint for CPR to
                 * suspend the system
                 */
                CALLB_CPR_SAFE_BEGIN(&cprinfo);

                /*
                 * hit the snooze bar
                 */
                (void) timeout(memscrub_wakeup, NULL, interval_sec * hz);

                /*
                 * go to sleep
                 */
                cv_wait(&memscrub_cv, &memscrub_lock);

                /* we need to goto work */
                CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock);

                mutex_exit(&memscrub_lock);

                do {
                        pgcnt_t pages = memscrub_span_pages;
                        uint64_t address = mlp_next_addr;

                        if (disable_memscrub || disable_memscrub_quietly)
                                break;

                        mutex_enter(&memscrub_lock);

                        /*
                         * Make sure we don't try to scan beyond the end of
                         * the current memlist.  If we would, then resize
                         * our scan target for this iteration, and prepare
                         * to read the next memlist entry on the next
                         * iteration.
                         */
                        reached_end = 0;
                        if (address + mmu_ptob(pages) >= mlp_last_addr) {
                                pages = mmu_btop(mlp_last_addr - address);
                                mlp = mlp->ml_next;
                                if (mlp == NULL) {
                                        reached_end = 1;
                                        mlp = memscrub_memlist;
                                }
                                mlp_next_addr = mlp->ml_address;
                                mlp_last_addr = mlp->ml_address + mlp->ml_size;
                        } else {
                                mlp_next_addr += mmu_ptob(pages);
                        }

                        mutex_exit(&memscrub_lock);

                        while (pages--) {
                                pfn_t pfn = btop(address);

                                /*
                                 * Without segkpm, the memscrubber cannot
                                 * be allowed to migrate across CPUs, as
                                 * the CPU-specific mapping of
                                 * memscrub_window would be incorrect.
                                 * With segkpm, switching CPUs is legal, but
                                 * inefficient.  We don't use
                                 * kpreempt_disable as it might hold a
                                 * higher priority thread (eg, RT) too long
                                 * off CPU.
                                 */
                                thread_affinity_set(curthread, CPU_CURRENT);
                                if (kpm_enable)
                                        memscrub_window = hat_kpm_pfn2va(pfn);
                                else
                                        hat_mempte_remap(pfn, memscrub_window,
                                            memscrub_pte,
                                            PROT_READ, HAT_LOAD_NOCONSIST);

                                scan_memory(memscrub_window, PAGESIZE);

                                thread_affinity_clear(curthread);
                                address += MMU_PAGESIZE;
                        }

                        memscrub_scans_done++;
                } while (!reached_end && system_is_idle());
        }

memscrub_exit:

        if (!disable_memscrub_quietly)
                cmn_err(CE_NOTE, "Software memory scrubber exiting.");
        /*
         * We are about to bail, but don't have the memscrub_lock,
         * and it is needed for CALLB_CPR_EXIT.
         */
        mutex_enter(&memscrub_lock);
        CALLB_CPR_EXIT(&cprinfo);

        cv_destroy(&memscrub_cv);

        thread_exit();
}


/*
 * return 1 if we're MP and all the other CPUs are idle
 */
static int
system_is_idle()
{
        int cpu_id;
        int found = 0;

        if (1 == ncpus_online)
                return (0);

        for (cpu_id = 0; cpu_id < NCPU; ++cpu_id) {
                if (!cpu[cpu_id])
                        continue;

                found++;

                if (cpu[cpu_id]->cpu_thread != cpu[cpu_id]->cpu_idle_thread) {
                        if (CPU->cpu_id == cpu_id &&
                            CPU->cpu_disp->disp_nrunnable == 0)
                                continue;
                        return (0);
                }

                if (found == ncpus)
                        break;
        }
        return (1);
}

/*
 * add a span to the memscrub list
 */
static int
memscrub_add_span(uint64_t start, uint64_t bytes)
{
        struct memlist *dst;
        struct memlist *prev, *next;
        uint64_t end = start + bytes - 1;
        int retval = 0;

        mutex_enter(&memscrub_lock);

#ifdef MEMSCRUB_DEBUG
        memscrub_printmemlist("memscrub_memlist before", memscrub_memlist);
        cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
        cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx"
            " size: 0x%llx\n", start, bytes);
#endif /* MEMSCRUB_DEBUG */

        /*
         * Scan through the list to find the proper place to install it.
         */
        prev = NULL;
        next = memscrub_memlist;
        while (next) {
                uint64_t ns = next->ml_address;
                uint64_t ne = next->ml_address + next->ml_size - 1;

                /*
                 * If this span overlaps with an existing span, then
                 * something has gone horribly wrong with the phys_install
                 * list.  In fact, I'm surprised we made it this far.
                 */
                if ((start >= ns && start <= ne) || (end >= ns && end <= ne) ||
                    (start < ns && end > ne))
                        panic("memscrub found overlapping memory ranges "
                            "(0x%p-0x%p) and (0x%p-0x%p)",
                            (void *)(uintptr_t)start, (void *)(uintptr_t)end,
                            (void *)(uintptr_t)ns, (void *)(uintptr_t)ne);

                /*
                 * New span can be appended to an existing one.
                 */
                if (start == ne + 1) {
                        next->ml_size += bytes;
                        goto add_done;
                }

                /*
                 * New span can be prepended to an existing one.
                 */
                if (end + 1 == ns) {
                        next->ml_size += bytes;
                        next->ml_address = start;
                        goto add_done;
                }

                /*
                 * If the next span has a higher start address than the new
                 * one, then we have found the right spot for our
                 * insertion.
                 */
                if (ns > start)
                        break;

                prev = next;
                next = next->ml_next;
        }

        /*
         * allocate a new struct memlist
         */
        dst = kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);
        if (dst == NULL) {
                retval = -1;
                goto add_done;
        }
        dst->ml_address = start;
        dst->ml_size = bytes;
        dst->ml_prev = prev;
        dst->ml_next = next;

        if (prev)
                prev->ml_next = dst;
        else
                memscrub_memlist = dst;

        if (next)
                next->ml_prev = dst;

add_done:

        if (retval != -1)
                memscrub_phys_pages += mmu_btop(bytes);

#ifdef MEMSCRUB_DEBUG
        memscrub_printmemlist("memscrub_memlist after", memscrub_memlist);
        cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
#endif /* MEMSCRUB_DEBUG */

        mutex_exit(&memscrub_lock);
        return (retval);
}