/*      $NetBSD: uvm_pdpolicy_clockpro.c,v 1.27 2022/04/12 20:27:56 andvar Exp $        */

/*-
 * Copyright (c)2005, 2006 YAMAMOTO Takashi,
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * CLOCK-Pro replacement policy:
 *      http://web.cse.ohio-state.edu/hpcs/WWW/HTML/publications/abs05-3.html
 *
 * approximation of the list of non-resident pages using hash:
 *      http://linux-mm.org/ClockProApproximation
 */

/* #define      CLOCKPRO_DEBUG */

#if defined(PDSIM)

#include "pdsim.h"

#else /* defined(PDSIM) */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.27 2022/04/12 20:27:56 andvar Exp $");

#include "opt_ddb.h"

#include <sys/param.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/hash.h>

#include <uvm/uvm.h>
#include <uvm/uvm_pdaemon.h>    /* for uvmpd_trylockowner */
#include <uvm/uvm_pdpolicy.h>
#include <uvm/uvm_pdpolicy_impl.h>

#if ((__STDC_VERSION__ - 0) >= 199901L)
#define DPRINTF(...)    /* nothing */
#define WARN(...)       printf(__VA_ARGS__)
#else /* ((__STDC_VERSION__ - 0) >= 199901L) */
#define DPRINTF(a...)   /* nothing */   /* GCC */
#define WARN(a...)      printf(a)
#endif /* ((__STDC_VERSION__ - 0) >= 199901L) */

#define dump(a)         /* nothing */

#undef  USEONCE2
#define LISTQ
#undef  ADAPTIVE

#endif /* defined(PDSIM) */

#if !defined(CLOCKPRO_COLDPCT)
#define CLOCKPRO_COLDPCT        10
#endif /* !defined(CLOCKPRO_COLDPCT) */

#define CLOCKPRO_COLDPCTMAX     90

#if !defined(CLOCKPRO_HASHFACTOR)
#define CLOCKPRO_HASHFACTOR     2
#endif /* !defined(CLOCKPRO_HASHFACTOR) */

#define CLOCKPRO_NEWQMIN        ((1024 * 1024) >> PAGE_SHIFT)   /* XXX */

int clockpro_hashfactor = CLOCKPRO_HASHFACTOR;

PDPOL_EVCNT_DEFINE(nresrecordobj)
PDPOL_EVCNT_DEFINE(nresrecordanon)
PDPOL_EVCNT_DEFINE(nreslookupobj)
PDPOL_EVCNT_DEFINE(nreslookupanon)
PDPOL_EVCNT_DEFINE(nresfoundobj)
PDPOL_EVCNT_DEFINE(nresfoundanon)
PDPOL_EVCNT_DEFINE(nresanonfree)
PDPOL_EVCNT_DEFINE(nresconflict)
PDPOL_EVCNT_DEFINE(nresoverwritten)
PDPOL_EVCNT_DEFINE(nreshandhot)

PDPOL_EVCNT_DEFINE(hhottakeover)
PDPOL_EVCNT_DEFINE(hhotref)
PDPOL_EVCNT_DEFINE(hhotunref)
PDPOL_EVCNT_DEFINE(hhotcold)
PDPOL_EVCNT_DEFINE(hhotcoldtest)

PDPOL_EVCNT_DEFINE(hcoldtakeover)
PDPOL_EVCNT_DEFINE(hcoldref)
PDPOL_EVCNT_DEFINE(hcoldunref)
PDPOL_EVCNT_DEFINE(hcoldreftest)
PDPOL_EVCNT_DEFINE(hcoldunreftest)
PDPOL_EVCNT_DEFINE(hcoldunreftestspeculative)
PDPOL_EVCNT_DEFINE(hcoldhot)

PDPOL_EVCNT_DEFINE(speculativeenqueue)
PDPOL_EVCNT_DEFINE(speculativehit1)
PDPOL_EVCNT_DEFINE(speculativehit2)
PDPOL_EVCNT_DEFINE(speculativemiss)

PDPOL_EVCNT_DEFINE(locksuccess)
PDPOL_EVCNT_DEFINE(lockfail)

#define PQ_REFERENCED   0x000000010
#define PQ_HOT          0x000000020
#define PQ_TEST         0x000000040
#define PQ_INITIALREF   0x000000080
#define PQ_QMASK        0x000000700
#define PQ_QFACTOR      0x000000100
#define PQ_SPECULATIVE  0x000000800

#define CLOCKPRO_NOQUEUE        0
#define CLOCKPRO_NEWQ           1       /* small queue to clear initial ref. */
#if defined(LISTQ)
#define CLOCKPRO_COLDQ          2
#define CLOCKPRO_HOTQ           3
#else /* defined(LISTQ) */
#define CLOCKPRO_COLDQ          (2 + coldqidx)  /* XXX */
#define CLOCKPRO_HOTQ           (3 - coldqidx)  /* XXX */
#endif /* defined(LISTQ) */
#define CLOCKPRO_LISTQ          4
#define CLOCKPRO_NQUEUE         4

static bool     uvmpdpol_pagerealize_locked(struct vm_page *);

static inline void
clockpro_setq(struct vm_page *pg, int qidx)
{
        KASSERT(qidx >= CLOCKPRO_NOQUEUE);
        KASSERT(qidx <= CLOCKPRO_NQUEUE);

        pg->pqflags = (pg->pqflags & ~PQ_QMASK) | (qidx * PQ_QFACTOR);
}

static inline int
clockpro_getq(struct vm_page *pg)
{
        int qidx;

        qidx = (pg->pqflags & PQ_QMASK) / PQ_QFACTOR;
        KASSERT(qidx >= CLOCKPRO_NOQUEUE);
        KASSERT(qidx <= CLOCKPRO_NQUEUE);
        return qidx;
}

typedef struct {
        struct pglist q_q;
        int q_len;
} pageq_t;

struct clockpro_state {
        kmutex_t lock;
        int s_npages;
        int s_coldtarget;
        int s_ncold;

        int s_newqlenmax;
        pageq_t s_q[CLOCKPRO_NQUEUE];

        struct uvm_pctparam s_coldtargetpct;
};

static pageq_t *
clockpro_queue(struct clockpro_state *s, int qidx)
{

        KASSERT(CLOCKPRO_NOQUEUE < qidx);
        KASSERT(qidx <= CLOCKPRO_NQUEUE);

        return &s->s_q[qidx - 1];
}

#if !defined(LISTQ)

static int coldqidx;

static void
clockpro_switchqueue(void)
{

        coldqidx = 1 - coldqidx;
}

#endif /* !defined(LISTQ) */

static struct clockpro_state clockpro __cacheline_aligned;
static struct clockpro_scanstate {
        int ss_nscanned;
} scanstate;

/* ---------------------------------------- */

static void
pageq_init(pageq_t *q)
{

        TAILQ_INIT(&q->q_q);
        q->q_len = 0;
}

static int
pageq_len(const pageq_t *q)
{

        return q->q_len;
}

static struct vm_page *
pageq_first(const pageq_t *q)
{

        return TAILQ_FIRST(&q->q_q);
}

static void
pageq_insert_tail(pageq_t *q, struct vm_page *pg)
{

        TAILQ_INSERT_TAIL(&q->q_q, pg, pdqueue);
        q->q_len++;
}

#if defined(LISTQ)
static void
pageq_insert_head(pageq_t *q, struct vm_page *pg)
{

        TAILQ_INSERT_HEAD(&q->q_q, pg, pdqueue);
        q->q_len++;
}
#endif

static void
pageq_remove(pageq_t *q, struct vm_page *pg)
{

#if 1
        KASSERT(clockpro_queue(&clockpro, clockpro_getq(pg)) == q);
#endif
        KASSERT(q->q_len > 0);
        TAILQ_REMOVE(&q->q_q, pg, pdqueue);
        q->q_len--;
}

static struct vm_page *
pageq_remove_head(pageq_t *q)
{
        struct vm_page *pg;

        pg = TAILQ_FIRST(&q->q_q);
        if (pg == NULL) {
                KASSERT(q->q_len == 0);
                return NULL;
        }
        pageq_remove(q, pg);
        return pg;
}

/* ---------------------------------------- */

static void
clockpro_insert_tail(struct clockpro_state *s, int qidx, struct vm_page *pg)
{
        pageq_t *q = clockpro_queue(s, qidx);
        
        clockpro_setq(pg, qidx);
        pageq_insert_tail(q, pg);
}

#if defined(LISTQ)
static void
clockpro_insert_head(struct clockpro_state *s, int qidx, struct vm_page *pg)
{
        pageq_t *q = clockpro_queue(s, qidx);
        
        clockpro_setq(pg, qidx);
        pageq_insert_head(q, pg);
}

#endif
/* ---------------------------------------- */

typedef uint32_t nonres_cookie_t;
#define NONRES_COOKIE_INVAL     0

typedef uintptr_t objid_t;

/*
 * XXX maybe these hash functions need reconsideration,
 * given that hash distribution is critical here.
 */

static uint32_t
pageidentityhash1(objid_t obj, off_t idx)
{
        uint32_t hash = HASH32_BUF_INIT;

#if 1
        hash = hash32_buf(&idx, sizeof(idx), hash);
        hash = hash32_buf(&obj, sizeof(obj), hash);
#else
        hash = hash32_buf(&obj, sizeof(obj), hash);
        hash = hash32_buf(&idx, sizeof(idx), hash);
#endif
        return hash;
}

static uint32_t
pageidentityhash2(objid_t obj, off_t idx)
{
        uint32_t hash = HASH32_BUF_INIT;

        hash = hash32_buf(&obj, sizeof(obj), hash);
        hash = hash32_buf(&idx, sizeof(idx), hash);
        return hash;
}

static nonres_cookie_t
calccookie(objid_t obj, off_t idx)
{
        uint32_t hash = pageidentityhash2(obj, idx);
        nonres_cookie_t cookie = hash;

        if (__predict_false(cookie == NONRES_COOKIE_INVAL)) {
                cookie++; /* XXX */
        }
        return cookie;
}

#define BUCKETSIZE      14
struct bucket {
        int cycle;
        int cur;
        nonres_cookie_t pages[BUCKETSIZE];
};
static int cycle_target;
static int cycle_target_frac;

static struct bucket static_bucket;
static struct bucket *buckets = &static_bucket;
static size_t hashsize = 1;

static int coldadj;
#define COLDTARGET_ADJ(d)       coldadj += (d)

#if defined(PDSIM)

static void *
clockpro_hashalloc(int n)
{
        size_t allocsz = sizeof(*buckets) * n;

        return malloc(allocsz);
}

static void
clockpro_hashfree(void *p, int n)
{

        free(p);
}

#else /* defined(PDSIM) */

static void *
clockpro_hashalloc(int n)
{
        size_t allocsz = round_page(sizeof(*buckets) * n);

        return (void *)uvm_km_alloc(kernel_map, allocsz, 0, UVM_KMF_WIRED);
}

static void
clockpro_hashfree(void *p, int n)
{
        size_t allocsz = round_page(sizeof(*buckets) * n);

        uvm_km_free(kernel_map, (vaddr_t)p, allocsz, UVM_KMF_WIRED);
}

#endif /* defined(PDSIM) */

static void
clockpro_hashinit(uint64_t n)
{
        struct bucket *newbuckets;
        struct bucket *oldbuckets;
        size_t sz;
        size_t oldsz;
        int i;

        sz = howmany(n, BUCKETSIZE);
        sz *= clockpro_hashfactor;
        newbuckets = clockpro_hashalloc(sz);
        if (newbuckets == NULL) {
                panic("%s: allocation failure", __func__);
        }
        for (i = 0; i < sz; i++) {
                struct bucket *b = &newbuckets[i];
                int j;

                b->cycle = cycle_target;
                b->cur = 0;
                for (j = 0; j < BUCKETSIZE; j++) {
                        b->pages[j] = NONRES_COOKIE_INVAL;
                }
        }
        /* XXX lock */
        oldbuckets = buckets;
        oldsz = hashsize;
        buckets = newbuckets;
        hashsize = sz;
        /* XXX unlock */
        if (oldbuckets != &static_bucket) {
                clockpro_hashfree(oldbuckets, oldsz);
        }
}

static struct bucket *
nonresident_getbucket(objid_t obj, off_t idx)
{
        uint32_t hash;

        hash = pageidentityhash1(obj, idx);
        return &buckets[hash % hashsize];
}

static void
nonresident_rotate(struct bucket *b)
{
        const int target = cycle_target;
        const int cycle = b->cycle;
        int cur;
        int todo;

        todo = target - cycle;
        if (todo >= BUCKETSIZE * 2) {
                todo = (todo % BUCKETSIZE) + BUCKETSIZE;
        }
        cur = b->cur;
        while (todo > 0) {
                if (b->pages[cur] != NONRES_COOKIE_INVAL) {
                        PDPOL_EVCNT_INCR(nreshandhot);
                        COLDTARGET_ADJ(-1);
                }
                b->pages[cur] = NONRES_COOKIE_INVAL;
                cur++;
                if (cur == BUCKETSIZE) {
                        cur = 0;
                }
                todo--;
        }
        b->cycle = target;
        b->cur = cur;
}

static bool
nonresident_lookupremove(objid_t obj, off_t idx)
{
        struct bucket *b = nonresident_getbucket(obj, idx);
        nonres_cookie_t cookie = calccookie(obj, idx);
        int i;

        nonresident_rotate(b);
        for (i = 0; i < BUCKETSIZE; i++) {
                if (b->pages[i] == cookie) {
                        b->pages[i] = NONRES_COOKIE_INVAL;
                        return true;
                }
        }
        return false;
}

static objid_t
pageobj(struct vm_page *pg)
{
        const void *obj;

        /*
         * XXX object pointer is often freed and reused for unrelated object.
         * for vnodes, it would be better to use something like
         * a hash of fsid/fileid/generation.
         */

        obj = pg->uobject;
        if (obj == NULL) {
                obj = pg->uanon;
                KASSERT(obj != NULL);
        }
        return (objid_t)obj;
}

static off_t
pageidx(struct vm_page *pg)
{

        KASSERT((pg->offset & PAGE_MASK) == 0);
        return pg->offset >> PAGE_SHIFT;
}

static bool
nonresident_pagelookupremove(struct vm_page *pg)
{
        bool found = nonresident_lookupremove(pageobj(pg), pageidx(pg));

        if (pg->uobject) {
                PDPOL_EVCNT_INCR(nreslookupobj);
        } else {
                PDPOL_EVCNT_INCR(nreslookupanon);
        }
        if (found) {
                if (pg->uobject) {
                        PDPOL_EVCNT_INCR(nresfoundobj);
                } else {
                        PDPOL_EVCNT_INCR(nresfoundanon);
                }
        }
        return found;
}

static void
nonresident_pagerecord(struct vm_page *pg)
{
        objid_t obj = pageobj(pg);
        off_t idx = pageidx(pg);
        struct bucket *b = nonresident_getbucket(obj, idx);
        nonres_cookie_t cookie = calccookie(obj, idx);

#if defined(DEBUG)
        int i;

        for (i = 0; i < BUCKETSIZE; i++) {
                if (b->pages[i] == cookie) {
                        PDPOL_EVCNT_INCR(nresconflict);
                }
        }
#endif /* defined(DEBUG) */

        if (pg->uobject) {
                PDPOL_EVCNT_INCR(nresrecordobj);
        } else {
                PDPOL_EVCNT_INCR(nresrecordanon);
        }
        nonresident_rotate(b);
        if (b->pages[b->cur] != NONRES_COOKIE_INVAL) {
                PDPOL_EVCNT_INCR(nresoverwritten);
                COLDTARGET_ADJ(-1);
        }
        b->pages[b->cur] = cookie;
        b->cur = (b->cur + 1) % BUCKETSIZE;
}

/* ---------------------------------------- */

#if defined(CLOCKPRO_DEBUG)
static void
check_sanity(void)
{
}
#else /* defined(CLOCKPRO_DEBUG) */
#define check_sanity()  /* nothing */
#endif /* defined(CLOCKPRO_DEBUG) */

static void
clockpro_reinit(void)
{

        KASSERT(mutex_owned(&clockpro.lock));

        clockpro_hashinit(uvmexp.npages);
}

static void
clockpro_init(void)
{
        struct clockpro_state *s = &clockpro;
        int i;

        mutex_init(&s->lock, MUTEX_DEFAULT, IPL_NONE);
        for (i = 0; i < CLOCKPRO_NQUEUE; i++) {
                pageq_init(&s->s_q[i]);
        }
        s->s_newqlenmax = 1;
        s->s_coldtarget = 1;
        uvm_pctparam_init(&s->s_coldtargetpct, CLOCKPRO_COLDPCT, NULL);
}

static void
clockpro_tune(void)
{
        struct clockpro_state *s = &clockpro;
        int coldtarget;

        KASSERT(mutex_owned(&s->lock));

#if defined(ADAPTIVE)
        int coldmax = s->s_npages * CLOCKPRO_COLDPCTMAX / 100;
        int coldmin = 1;

        coldtarget = s->s_coldtarget;
        if (coldtarget + coldadj < coldmin) {
                coldadj = coldmin - coldtarget;
        } else if (coldtarget + coldadj > coldmax) {
                coldadj = coldmax - coldtarget;
        }
        coldtarget += coldadj;
#else /* defined(ADAPTIVE) */
        coldtarget = UVM_PCTPARAM_APPLY(&s->s_coldtargetpct, s->s_npages);
        if (coldtarget < 1) {
                coldtarget = 1;
        }
#endif /* defined(ADAPTIVE) */

        s->s_coldtarget = coldtarget;
        s->s_newqlenmax = coldtarget / 4;
        if (s->s_newqlenmax < CLOCKPRO_NEWQMIN) {
                s->s_newqlenmax = CLOCKPRO_NEWQMIN;
        }
}

static void
clockpro_movereferencebit(struct vm_page *pg, bool locked)
{
        kmutex_t *lock;
        bool referenced;

        KASSERT(mutex_owned(&clockpro.lock));
        KASSERT(!locked || uvm_page_owner_locked_p(pg, false));
        if (!locked) {
                /*
                 * acquire interlock to stabilize page identity.
                 * if we have caught the page in a state of flux
                 * and it should be dequeued, abort.  it will be
                 * dequeued later.
                 */
                mutex_enter(&pg->interlock);
                if ((pg->uobject == NULL && pg->uanon == NULL) ||
                    pg->wire_count > 0) {
                        mutex_exit(&pg->interlock);
                        PDPOL_EVCNT_INCR(lockfail);
                        return;
                }
                mutex_exit(&clockpro.lock);     /* XXX */
                lock = uvmpd_trylockowner(pg);
                /* pg->interlock now dropped */
                mutex_enter(&clockpro.lock);    /* XXX */
                if (lock == NULL) {
                        /*
                         * XXXuvmplock
                         */
                        PDPOL_EVCNT_INCR(lockfail);
                        return;
                }
                PDPOL_EVCNT_INCR(locksuccess);
        }
        referenced = pmap_clear_reference(pg);
        if (!locked) {
                mutex_exit(lock);
        }
        if (referenced) {
                pg->pqflags |= PQ_REFERENCED;
        }
}

static void
clockpro_clearreferencebit(struct vm_page *pg, bool locked)
{

        KASSERT(mutex_owned(&clockpro.lock));

        clockpro_movereferencebit(pg, locked);
        pg->pqflags &= ~PQ_REFERENCED;
}

static void
clockpro___newqrotate(int len)
{
        struct clockpro_state * const s = &clockpro;
        pageq_t * const newq = clockpro_queue(s, CLOCKPRO_NEWQ);
        struct vm_page *pg;

        KASSERT(mutex_owned(&s->lock));

        while (pageq_len(newq) > len) {
                pg = pageq_remove_head(newq);
                KASSERT(pg != NULL);
                KASSERT(clockpro_getq(pg) == CLOCKPRO_NEWQ);
                if ((pg->pqflags & PQ_INITIALREF) != 0) {
                        clockpro_clearreferencebit(pg, false);
                        pg->pqflags &= ~PQ_INITIALREF;
                }
                /* place at the list head */
                clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg);
        }
}

static void
clockpro_newqrotate(void)
{
        struct clockpro_state * const s = &clockpro;

        KASSERT(mutex_owned(&s->lock));

        check_sanity();
        clockpro___newqrotate(s->s_newqlenmax);
        check_sanity();
}

static void
clockpro_newqflush(int n)
{

        KASSERT(mutex_owned(&clockpro.lock));

        check_sanity();
        clockpro___newqrotate(n);
        check_sanity();
}

static void
clockpro_newqflushone(void)
{
        struct clockpro_state * const s = &clockpro;

        KASSERT(mutex_owned(&s->lock));

        clockpro_newqflush(
            MAX(pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) - 1, 0));
}

/*
 * our "tail" is called "list-head" in the paper.
 */

static void
clockpro___enqueuetail(struct vm_page *pg)
{
        struct clockpro_state * const s = &clockpro;

        KASSERT(mutex_owned(&s->lock));
        KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE);

        check_sanity();
#if !defined(USEONCE2)
        clockpro_insert_tail(s, CLOCKPRO_NEWQ, pg);
        clockpro_newqrotate();
#else /* !defined(USEONCE2) */
#if defined(LISTQ)
        KASSERT((pg->pqflags & PQ_REFERENCED) == 0);
#endif /* defined(LISTQ) */
        clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg);
#endif /* !defined(USEONCE2) */
        check_sanity();
}

static void
clockpro_pageenqueue(struct vm_page *pg)
{
        struct clockpro_state * const s = &clockpro;
        bool hot;
        bool speculative = (pg->pqflags & PQ_SPECULATIVE) != 0; /* XXX */

        KASSERT((~pg->pqflags & (PQ_INITIALREF|PQ_SPECULATIVE)) != 0);
        KASSERT(mutex_owned(&s->lock));
        check_sanity();
        KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE);
        s->s_npages++;
        pg->pqflags &= ~(PQ_HOT|PQ_TEST);
        if (speculative) {
                hot = false;
                PDPOL_EVCNT_INCR(speculativeenqueue);
        } else {
                hot = nonresident_pagelookupremove(pg);
                if (hot) {
                        COLDTARGET_ADJ(1);
                }
        }

        /*
         * consider mmap'ed file:
         *
         * - read-ahead enqueues a page.
         *
         * - on the following read-ahead hit, the fault handler activates it.
         *
         * - finally, the userland code which caused the above fault
         *   actually accesses the page.  it makes its reference bit set.
         *
         * we want to count the above as a single access, rather than
         * three accesses with short reuse distances.
         */

#if defined(USEONCE2)
        pg->pqflags &= ~PQ_INITIALREF;
        if (hot) {
                pg->pqflags |= PQ_TEST;
        }
        s->s_ncold++;
        clockpro_clearreferencebit(pg, false);
        clockpro___enqueuetail(pg);
#else /* defined(USEONCE2) */
        if (speculative) {
                s->s_ncold++;
        } else if (hot) {
                pg->pqflags |= PQ_HOT;
        } else {
                pg->pqflags |= PQ_TEST;
                s->s_ncold++;
        }
        clockpro___enqueuetail(pg);
#endif /* defined(USEONCE2) */
        KASSERT(s->s_ncold <= s->s_npages);
}

static pageq_t *
clockpro_pagequeue(struct vm_page *pg)
{
        struct clockpro_state * const s = &clockpro;
        int qidx;

        KASSERT(mutex_owned(&s->lock));

        qidx = clockpro_getq(pg);
        KASSERT(qidx != CLOCKPRO_NOQUEUE);

        return clockpro_queue(s, qidx);
}

static void
clockpro_pagedequeue(struct vm_page *pg)
{
        struct clockpro_state * const s = &clockpro;
        pageq_t *q;

        KASSERT(mutex_owned(&s->lock));

        KASSERT(s->s_npages > 0);
        check_sanity();
        q = clockpro_pagequeue(pg);
        pageq_remove(q, pg);
        check_sanity();
        clockpro_setq(pg, CLOCKPRO_NOQUEUE);
        if ((pg->pqflags & PQ_HOT) == 0) {
                KASSERT(s->s_ncold > 0);
                s->s_ncold--;
        }
        KASSERT(s->s_npages > 0);
        s->s_npages--;
        check_sanity();
}

static void
clockpro_pagerequeue(struct vm_page *pg)
{
        struct clockpro_state * const s = &clockpro;
        int qidx;

        KASSERT(mutex_owned(&s->lock));

        qidx = clockpro_getq(pg);
        KASSERT(qidx == CLOCKPRO_HOTQ || qidx == CLOCKPRO_COLDQ);
        pageq_remove(clockpro_queue(s, qidx), pg);
        check_sanity();
        clockpro_setq(pg, CLOCKPRO_NOQUEUE);

        clockpro___enqueuetail(pg);
}

static void
handhot_endtest(struct vm_page *pg)
{

        KASSERT(mutex_owned(&clockpro.lock));

        KASSERT((pg->pqflags & PQ_HOT) == 0);
        if ((pg->pqflags & PQ_TEST) != 0) {
                PDPOL_EVCNT_INCR(hhotcoldtest);
                COLDTARGET_ADJ(-1);
                pg->pqflags &= ~PQ_TEST;
        } else {
                PDPOL_EVCNT_INCR(hhotcold);
        }
}

static void
handhot_advance(void)
{
        struct clockpro_state * const s = &clockpro;
        struct vm_page *pg;
        pageq_t *hotq;
        int hotqlen;

        KASSERT(mutex_owned(&s->lock));

        clockpro_tune();

        dump("hot called");
        if (s->s_ncold >= s->s_coldtarget) {
                return;
        }
        hotq = clockpro_queue(s, CLOCKPRO_HOTQ);
again:
        pg = pageq_first(hotq);
        if (pg == NULL) {
                DPRINTF("%s: HHOT TAKEOVER\n", __func__);
                dump("hhottakeover");
                PDPOL_EVCNT_INCR(hhottakeover);
#if defined(LISTQ)
                while (/* CONSTCOND */ 1) {
                        pageq_t *coldq = clockpro_queue(s, CLOCKPRO_COLDQ);

                        pg = pageq_first(coldq);
                        if (pg == NULL) {
                                clockpro_newqflushone();
                                pg = pageq_first(coldq);
                                if (pg == NULL) {
                                        WARN("hhot: no page?\n");
                                        return;
                                }
                        }
                        KASSERT(clockpro_pagequeue(pg) == coldq);
                        pageq_remove(coldq, pg);
                        check_sanity();
                        if ((pg->pqflags & PQ_HOT) == 0) {
                                handhot_endtest(pg);
                                clockpro_insert_tail(s, CLOCKPRO_LISTQ, pg);
                        } else {
                                clockpro_insert_head(s, CLOCKPRO_HOTQ, pg);
                                break;
                        }
                }
#else /* defined(LISTQ) */
                clockpro_newqflush(0); /* XXX XXX */
                clockpro_switchqueue();
                hotq = clockpro_queue(s, CLOCKPRO_HOTQ);
                goto again;
#endif /* defined(LISTQ) */
        }

        KASSERT(clockpro_pagequeue(pg) == hotq);

        /*
         * terminate test period of nonresident pages by cycling them.
         */

        cycle_target_frac += BUCKETSIZE;
        hotqlen = pageq_len(hotq);
        while (cycle_target_frac >= hotqlen) {
                cycle_target++;
                cycle_target_frac -= hotqlen;
        }

        if ((pg->pqflags & PQ_HOT) == 0) {
#if defined(LISTQ)
                panic("cold page in hotq: %p", pg);
#else /* defined(LISTQ) */
                handhot_endtest(pg);
                goto next;
#endif /* defined(LISTQ) */
        }
        KASSERT((pg->pqflags & PQ_TEST) == 0);
        KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
        KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0);

        /*
         * once we met our target,
         * stop at a hot page so that no cold pages in test period
         * have larger recency than any hot pages.
         */

        if (s->s_ncold >= s->s_coldtarget) {
                dump("hot done");
                return;
        }
        clockpro_movereferencebit(pg, false);
        if ((pg->pqflags & PQ_REFERENCED) == 0) {
                PDPOL_EVCNT_INCR(hhotunref);
                uvmexp.pddeact++;
                pg->pqflags &= ~PQ_HOT;
                clockpro.s_ncold++;
                KASSERT(s->s_ncold <= s->s_npages);
        } else {
                PDPOL_EVCNT_INCR(hhotref);
        }
        pg->pqflags &= ~PQ_REFERENCED;
#if !defined(LISTQ)
next:
#endif /* !defined(LISTQ) */
        clockpro_pagerequeue(pg);
        dump("hot");
        goto again;
}

static struct vm_page *
handcold_advance(void)
{
        struct clockpro_state * const s = &clockpro;
        struct vm_page *pg;

        KASSERT(mutex_owned(&s->lock));

        for (;;) {
#if defined(LISTQ)
                pageq_t *listq = clockpro_queue(s, CLOCKPRO_LISTQ);
#endif /* defined(LISTQ) */
                pageq_t *coldq;

                clockpro_newqrotate();
                handhot_advance();
#if defined(LISTQ)
                pg = pageq_first(listq);
                if (pg != NULL) {
                        KASSERT(clockpro_getq(pg) == CLOCKPRO_LISTQ);
                        KASSERT((pg->pqflags & PQ_TEST) == 0);
                        KASSERT((pg->pqflags & PQ_HOT) == 0);
                        KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
                        pageq_remove(listq, pg);
                        check_sanity();
                        clockpro_insert_head(s, CLOCKPRO_COLDQ, pg); /* XXX */
                        goto gotcold;
                }
#endif /* defined(LISTQ) */
                check_sanity();
                coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
                pg = pageq_first(coldq);
                if (pg == NULL) {
                        clockpro_newqflushone();
                        pg = pageq_first(coldq);
                }
                if (pg == NULL) {
                        DPRINTF("%s: HCOLD TAKEOVER\n", __func__);
                        dump("hcoldtakeover");
                        PDPOL_EVCNT_INCR(hcoldtakeover);
                        KASSERT(
                            pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) == 0);
#if defined(LISTQ)
                        KASSERT(
                            pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)) == 0);
#else /* defined(LISTQ) */
                        clockpro_switchqueue();
                        coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
                        pg = pageq_first(coldq);
#endif /* defined(LISTQ) */
                }
                if (pg == NULL) {
                        WARN("hcold: no page?\n");
                        return NULL;
                }
                KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
                if ((pg->pqflags & PQ_HOT) != 0) {
                        PDPOL_EVCNT_INCR(hcoldhot);
                        pageq_remove(coldq, pg);
                        clockpro_insert_tail(s, CLOCKPRO_HOTQ, pg);
                        check_sanity();
                        KASSERT((pg->pqflags & PQ_TEST) == 0);
                        uvmexp.pdscans++;
                        continue;
                }
#if defined(LISTQ)
gotcold:
#endif /* defined(LISTQ) */
                KASSERT((pg->pqflags & PQ_HOT) == 0);
                uvmexp.pdscans++;
                clockpro_movereferencebit(pg, false);
                if ((pg->pqflags & PQ_SPECULATIVE) != 0) {
                        KASSERT((pg->pqflags & PQ_TEST) == 0);
                        if ((pg->pqflags & PQ_REFERENCED) != 0) {
                                PDPOL_EVCNT_INCR(speculativehit2);
                                pg->pqflags &= ~(PQ_SPECULATIVE|PQ_REFERENCED);
                                clockpro_pagedequeue(pg);
                                clockpro_pageenqueue(pg);
                                continue;
                        }
                        PDPOL_EVCNT_INCR(speculativemiss);
                }
                switch (pg->pqflags & (PQ_REFERENCED|PQ_TEST)) {
                case PQ_TEST:
                        PDPOL_EVCNT_INCR(hcoldunreftest);
                        nonresident_pagerecord(pg);
                        goto gotit;
                case 0:
                        PDPOL_EVCNT_INCR(hcoldunref);
gotit:
                        KASSERT(s->s_ncold > 0);
                        clockpro_pagerequeue(pg); /* XXX */
                        dump("cold done");
                        /* XXX "pg" is still in queue */
                        handhot_advance();
                        goto done;

                case PQ_REFERENCED|PQ_TEST:
                        PDPOL_EVCNT_INCR(hcoldreftest);
                        s->s_ncold--;
                        COLDTARGET_ADJ(1);
                        pg->pqflags |= PQ_HOT;
                        pg->pqflags &= ~PQ_TEST;
                        break;

                case PQ_REFERENCED:
                        PDPOL_EVCNT_INCR(hcoldref);
                        pg->pqflags |= PQ_TEST;
                        break;
                }
                pg->pqflags &= ~PQ_REFERENCED;
                uvmexp.pdreact++;
                /* move to the list head */
                clockpro_pagerequeue(pg);
                dump("cold");
        }
done:;
        return pg;
}

static void
uvmpdpol_pageactivate_locked(struct vm_page *pg)
{

        if (!uvmpdpol_pageisqueued_p(pg)) {
                KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0);
                pg->pqflags |= PQ_INITIALREF;
                clockpro_pageenqueue(pg);
        } else if ((pg->pqflags & PQ_SPECULATIVE)) {
                PDPOL_EVCNT_INCR(speculativehit1);
                pg->pqflags &= ~PQ_SPECULATIVE;
                pg->pqflags |= PQ_INITIALREF;
                clockpro_pagedequeue(pg);
                clockpro_pageenqueue(pg);
        }
        pg->pqflags |= PQ_REFERENCED;
}

void
uvmpdpol_pageactivate(struct vm_page *pg)
{

        uvmpdpol_set_intent(pg, PQ_INTENT_A);
}

static void
uvmpdpol_pagedeactivate_locked(struct vm_page *pg)
{

        clockpro_clearreferencebit(pg, true);
}

void
uvmpdpol_pagedeactivate(struct vm_page *pg)
{

        uvmpdpol_set_intent(pg, PQ_INTENT_I);
}

static void
uvmpdpol_pagedequeue_locked(struct vm_page *pg)
{

        if (!uvmpdpol_pageisqueued_p(pg)) {
                return;
        }
        clockpro_pagedequeue(pg);
        pg->pqflags &= ~(PQ_INITIALREF|PQ_SPECULATIVE);
}

void
uvmpdpol_pagedequeue(struct vm_page *pg)
{

        uvmpdpol_set_intent(pg, PQ_INTENT_D);
}

static void
uvmpdpol_pageenqueue_locked(struct vm_page *pg)
{

#if 1
        if (uvmpdpol_pageisqueued_p(pg)) {
                return;
        }
        clockpro_clearreferencebit(pg, true);
        pg->pqflags |= PQ_SPECULATIVE;
        clockpro_pageenqueue(pg);
#else
        uvmpdpol_pageactivate_locked(pg);
#endif
}

void
uvmpdpol_pageenqueue(struct vm_page *pg)
{

        uvmpdpol_set_intent(pg, PQ_INTENT_D);
}

static bool
uvmpdpol_pagerealize_locked(struct vm_page *pg)
{
        uint32_t pqflags;

        KASSERT(mutex_owned(&clockpro.lock));
        KASSERT(mutex_owned(&pg->interlock));

        /* XXX this needs to be called from elsewhere, like uvmpdpol_clock. */

        pqflags = pg->pqflags;
        pq->pqflags &= ~(PQ_INTENT_SET | PQ_INTENT_QUEUED);
        switch (pqflags & (PQ_INTENT_MASK | PQ_INTENT_SET)) {
        case PQ_INTENT_A | PQ_INTENT_SET:
                uvmpdpol_pageactivate_locked(pg);
                return true;
        case PQ_INTENT_E | PQ_INTENT_SET:
                uvmpdpol_pageenqueue_locked(pg);
                return true;
        case PQ_INTENT_I | PQ_INTENT_SET:
                uvmpdpol_pagedeactivate_locked(pg);
                return true;
        case PQ_INTENT_D | PQ_INTENT_SET:
                uvmpdpol_pagedequeue_locked(pg);
                return true;
        default:
                return false;
        }
}

void
uvmpdpol_pagerealize(struct vm_page *pg)
{
        struct clockpro_state * const s = &clockpro;

        mutex_enter(&s->lock);
        uvmpdpol_pagerealize_locked(pg);
        mutex_exit(&s->lock);
}

void
uvmpdpol_anfree(struct vm_anon *an)
{
        struct clockpro_state * const s = &clockpro;

        KASSERT(an->an_page == NULL);
        mutex_enter(&s->lock);
        if (nonresident_lookupremove((objid_t)an, 0)) {
                PDPOL_EVCNT_INCR(nresanonfree);
        }
        mutex_exit(&s->lock);
}

void
uvmpdpol_init(void)
{

        clockpro_init();
}

void
uvmpdpol_reinit(void)
{
        struct clockpro_state * const s = &clockpro;

        mutex_enter(&s->lock);
        clockpro_reinit();
        mutex_exit(&s->lock);
}

void
uvmpdpol_estimatepageable(int *active, int *inactive)
{
        struct clockpro_state * const s = &clockpro;

        /*
         * Don't take any locks here.  This can be called from DDB, and in
         * any case the numbers are stale the instant the lock is dropped,
         * so it just doesn't matter.
         */
        if (active) {
                *active = s->s_npages - s->s_ncold;
        }
        if (inactive) {
                *inactive = s->s_ncold;
        }
}

bool
uvmpdpol_pageisqueued_p(struct vm_page *pg)
{

        /* Unlocked check OK due to page lifecycle. */
        return clockpro_getq(pg) != CLOCKPRO_NOQUEUE;
}

bool
uvmpdpol_pageactivate_p(struct vm_page *pg)
{

        /* For now, no heuristic, always receive activations. */
        return true;
}

void
uvmpdpol_scaninit(void)
{
        struct clockpro_state * const s = &clockpro;
        struct clockpro_scanstate * const ss = &scanstate;

        mutex_enter(&s->lock);
        ss->ss_nscanned = 0;
        mutex_exit(&s->lock);
}

void
uvmpdpol_scanfini(void)
{

}

struct vm_page *
uvmpdpol_selectvictim(kmutex_t **plock)
{
        struct clockpro_state * const s = &clockpro;
        struct clockpro_scanstate * const ss = &scanstate;
        struct vm_page *pg;
        kmutex_t *lock = NULL;

        do {
                mutex_enter(&s->lock);
                if (ss->ss_nscanned > s->s_npages) {
                        DPRINTF("scan too much\n");
                        mutex_exit(&s->lock);
                        return NULL;
                }
                pg = handcold_advance();
                if (pg == NULL) {
                        mutex_exit(&s->lock);
                        break;
                }
                ss->ss_nscanned++;
                /*
                 * acquire interlock to stabilize page identity.
                 * if we have caught the page in a state of flux
                 * and it should be dequeued, do it now and then
                 * move on to the next.
                 */
                mutex_enter(&pg->interlock);
                if ((pg->uobject == NULL && pg->uanon == NULL) ||
                    pg->wire_count > 0) {
                        mutex_exit(&pg->interlock);
                        clockpro_pagedequeue(pg);
                        pg->pqflags &= ~(PQ_INITIALREF|PQ_SPECULATIVE);
                        continue;
                }
                mutex_exit(&s->lock);
                lock = uvmpd_trylockowner(pg);
                /* pg->interlock now dropped */
        } while (lock == NULL);
        *plock = lock;
        return pg;
}

static void
clockpro_dropswap(pageq_t *q, int *todo)
{
        struct vm_page *pg;
        kmutex_t *lock;

        KASSERT(mutex_owned(&clockpro.lock));

        TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pdqueue) {
                if (*todo <= 0) {
                        break;
                }
                if ((pg->pqflags & PQ_HOT) == 0) {
                        continue;
                }
                mutex_enter(&pg->interlock);
                if ((pg->flags & PG_SWAPBACKED) == 0) {
                        mutex_exit(&pg->interlock);
                        continue;
                }

                /*
                 * try to lock the object that owns the page.
                 */
                mutex_exit(&clockpro.lock);
                lock = uvmpd_trylockowner(pg);
                /* pg->interlock now released */
                mutex_enter(&clockpro.lock);
                if (lock == NULL) {
                        /* didn't get it - try the next page. */
                        /* XXXAD lost position in queue */
                        continue;
                }

                /*
                 * if there's a shortage of swap slots, try to free it.
                 */
                if ((pg->flags & PG_SWAPBACKED) != 0 &&
                    (pg->flags & PG_BUSY) == 0) {
                        if (uvmpd_dropswap(pg)) {
                                (*todo)--;
                        }
                }
                mutex_exit(lock);
        }
}

void
uvmpdpol_balancequeue(int swap_shortage)
{
        struct clockpro_state * const s = &clockpro;
        int todo = swap_shortage;

        if (todo == 0) {
                return;
        }

        /*
         * reclaim swap slots from hot pages
         */

        DPRINTF("%s: swap_shortage=%d\n", __func__, swap_shortage);

        mutex_enter(&s->lock);
        clockpro_dropswap(clockpro_queue(s, CLOCKPRO_NEWQ), &todo);
        clockpro_dropswap(clockpro_queue(s, CLOCKPRO_COLDQ), &todo);
        clockpro_dropswap(clockpro_queue(s, CLOCKPRO_HOTQ), &todo);
        mutex_exit(&s->lock);

        DPRINTF("%s: done=%d\n", __func__, swap_shortage - todo);
}

bool
uvmpdpol_needsscan_p(void)
{
        struct clockpro_state * const s = &clockpro;

        /* This must be an unlocked check: can be called from interrupt. */
        return s->s_ncold < s->s_coldtarget;
}

void
uvmpdpol_tune(void)
{
        struct clockpro_state * const s = &clockpro;

        mutex_enter(&s->lock);
        clockpro_tune();
        mutex_exit(&s->lock);
}

void
uvmpdpol_idle(void)
{

}

#if !defined(PDSIM)

#include <sys/sysctl.h> /* XXX SYSCTL_DESCR */

void
uvmpdpol_sysctlsetup(void)
{
#if !defined(ADAPTIVE)
        struct clockpro_state * const s = &clockpro;

        uvm_pctparam_createsysctlnode(&s->s_coldtargetpct, "coldtargetpct",
            SYSCTL_DESCR("Percentage cold target queue of the entire queue"));
#endif /* !defined(ADAPTIVE) */
}

#endif /* !defined(PDSIM) */

#if defined(DDB)

#if 0 /* XXXuvmplock */
#define _pmap_is_referenced(pg) pmap_is_referenced(pg)
#else
#define _pmap_is_referenced(pg) false
#endif

void clockpro_dump(void);

void
clockpro_dump(void)
{
        struct clockpro_state * const s = &clockpro;

        struct vm_page *pg;
        int ncold, nhot, ntest, nspeculative, ninitialref, nref;
        int newqlen, coldqlen, hotqlen, listqlen;

        newqlen = coldqlen = hotqlen = listqlen = 0;
        printf("npages=%d, ncold=%d, coldtarget=%d, newqlenmax=%d\n",
            s->s_npages, s->s_ncold, s->s_coldtarget, s->s_newqlenmax);

#define INITCOUNT()     \
        ncold = nhot = ntest = nspeculative = ninitialref = nref = 0

#define COUNT(pg)       \
        if ((pg->pqflags & PQ_HOT) != 0) { \
                nhot++; \
        } else { \
                ncold++; \
                if ((pg->pqflags & PQ_TEST) != 0) { \
                        ntest++; \
                } \
                if ((pg->pqflags & PQ_SPECULATIVE) != 0) { \
                        nspeculative++; \
                } \
                if ((pg->pqflags & PQ_INITIALREF) != 0) { \
                        ninitialref++; \
                } else if ((pg->pqflags & PQ_REFERENCED) != 0 || \
                    _pmap_is_referenced(pg)) { \
                        nref++; \
                } \
        }

#define PRINTCOUNT(name)        \
        printf("%s hot=%d, cold=%d, test=%d, speculative=%d, initialref=%d, " \
            "nref=%d\n", \
            (name), nhot, ncold, ntest, nspeculative, ninitialref, nref)

        INITCOUNT();
        TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_NEWQ)->q_q, pdqueue) {
                if (clockpro_getq(pg) != CLOCKPRO_NEWQ) {
                        printf("newq corrupt %p\n", pg);
                }
                COUNT(pg)
                newqlen++;
        }
        PRINTCOUNT("newq");

        INITCOUNT();
        TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_COLDQ)->q_q, pdqueue) {
                if (clockpro_getq(pg) != CLOCKPRO_COLDQ) {
                        printf("coldq corrupt %p\n", pg);
                }
                COUNT(pg)
                coldqlen++;
        }
        PRINTCOUNT("coldq");

        INITCOUNT();
        TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_HOTQ)->q_q, pdqueue) {
                if (clockpro_getq(pg) != CLOCKPRO_HOTQ) {
                        printf("hotq corrupt %p\n", pg);
                }
#if defined(LISTQ)
                if ((pg->pqflags & PQ_HOT) == 0) {
                        printf("cold page in hotq: %p\n", pg);
                }
#endif /* defined(LISTQ) */
                COUNT(pg)
                hotqlen++;
        }
        PRINTCOUNT("hotq");

        INITCOUNT();
        TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_LISTQ)->q_q, pdqueue) {
#if !defined(LISTQ)
                printf("listq %p\n", pg);
#endif /* !defined(LISTQ) */
                if (clockpro_getq(pg) != CLOCKPRO_LISTQ) {
                        printf("listq corrupt %p\n", pg);
                }
                COUNT(pg)
                listqlen++;
        }
        PRINTCOUNT("listq");

        printf("newqlen=%d/%d, coldqlen=%d/%d, hotqlen=%d/%d, listqlen=%d/%d\n",
            newqlen, pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)),
            coldqlen, pageq_len(clockpro_queue(s, CLOCKPRO_COLDQ)),
            hotqlen, pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)),
            listqlen, pageq_len(clockpro_queue(s, CLOCKPRO_LISTQ)));
}

#endif /* defined(DDB) */

#if defined(PDSIM)
#if defined(DEBUG)
static void
pdsim_dumpq(int qidx)
{
        struct clockpro_state * const s = &clockpro;
        pageq_t *q = clockpro_queue(s, qidx);
        struct vm_page *pg;

        TAILQ_FOREACH(pg, &q->q_q, pdqueue) {
                DPRINTF(" %" PRIu64 "%s%s%s%s%s%s",
                    pg->offset >> PAGE_SHIFT,
                    (pg->pqflags & PQ_HOT) ? "H" : "",
                    (pg->pqflags & PQ_TEST) ? "T" : "",
                    (pg->pqflags & PQ_REFERENCED) ? "R" : "",
                    _pmap_is_referenced(pg) ? "r" : "",
                    (pg->pqflags & PQ_INITIALREF) ? "I" : "",
                    (pg->pqflags & PQ_SPECULATIVE) ? "S" : ""
                    );
        }
}
#endif /* defined(DEBUG) */

void
pdsim_dump(const char *id)
{
#if defined(DEBUG)
        struct clockpro_state * const s = &clockpro;

        DPRINTF("  %s L(", id);
        pdsim_dumpq(CLOCKPRO_LISTQ);
        DPRINTF(" ) H(");
        pdsim_dumpq(CLOCKPRO_HOTQ);
        DPRINTF(" ) C(");
        pdsim_dumpq(CLOCKPRO_COLDQ);
        DPRINTF(" ) N(");
        pdsim_dumpq(CLOCKPRO_NEWQ);
        DPRINTF(" ) ncold=%d/%d, coldadj=%d\n",
            s->s_ncold, s->s_coldtarget, coldadj);
#endif /* defined(DEBUG) */
}
#endif /* defined(PDSIM) */