/*      $NetBSD: uvm_pager.c,v 1.136 2026/05/03 16:02:37 thorpej Exp $  */

/*
 * Copyright (c) 1997 Charles D. Cranor and Washington University.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp
 */

/*
 * uvm_pager.c: generic functions used to assist the pagers.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.136 2026/05/03 16:02:37 thorpej Exp $");

#include "opt_uvmhist.h"
#include "opt_readahead.h"
#include "opt_pagermap.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/atomic.h>
#include <sys/vnode.h>
#include <sys/buf.h>

#include <uvm/uvm.h>

/*
 * XXX
 * this is needed until the device strategy interface
 * is changed to do physically-addressed i/o.
 */

#ifndef PAGER_MAP_DEFAULT_SIZE
#define PAGER_MAP_DEFAULT_SIZE  (16 * 1024 * 1024)
#endif

#ifndef PAGER_MAP_SIZE
#define PAGER_MAP_SIZE  PAGER_MAP_DEFAULT_SIZE
#endif

size_t pager_map_size = PAGER_MAP_SIZE;

/*
 * list of uvm pagers in the system
 */

const struct uvm_pagerops * const uvmpagerops[] = {
        &aobj_pager,
        &uvm_deviceops,
        &uvm_vnodeops,
        &ubc_pager,
};

/*
 * the pager map: provides KVA for I/O
 */

struct vm_map *pager_map;               /* XXX */
kmutex_t pager_map_wanted_lock __cacheline_aligned;
bool pager_map_wanted;  /* locked by pager map */
static vaddr_t emergva;
static int emerg_ncolors;
static bool emerginuse;

void
uvm_pager_realloc_emerg(void)
{
        vaddr_t new_emergva, old_emergva;
        int old_emerg_ncolors;

        if (__predict_true(emergva != 0 && emerg_ncolors >= uvmexp.ncolors))
                return;

        KASSERT(!emerginuse);

        new_emergva = uvm_km_alloc(kernel_map,
            round_page(MAXPHYS) + ptoa(uvmexp.ncolors), ptoa(uvmexp.ncolors),
            UVM_KMF_VAONLY);

        KASSERT(new_emergva != 0);

        old_emergva = emergva;
        old_emerg_ncolors = emerg_ncolors;

        /*
         * don't support re-color in late boot anyway.
         */
        if (0) /* XXX */
                mutex_enter(&pager_map_wanted_lock);

        emergva = new_emergva;
        emerg_ncolors = uvmexp.ncolors;
        wakeup(&old_emergva);

        if (0) /* XXX */
                mutex_exit(&pager_map_wanted_lock);

        if (old_emergva)
                uvm_km_free(kernel_map, old_emergva,
                    round_page(MAXPHYS) + ptoa(old_emerg_ncolors),
                    UVM_KMF_VAONLY);
}

/*
 * uvm_pager_init: init pagers (at boot time)
 */

void
uvm_pager_init(void)
{
        u_int lcv;
        vaddr_t sva, eva;

        /*
         * init pager map
         */

        sva = 0;
        pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, pager_map_size, 0,
            false, NULL);
        mutex_init(&pager_map_wanted_lock, MUTEX_DEFAULT, IPL_NONE);
        pager_map_wanted = false;

        uvm_pager_realloc_emerg();

        /*
         * call pager init functions
         */
        for (lcv = 0 ; lcv < __arraycount(uvmpagerops); lcv++) {
                if (uvmpagerops[lcv]->pgo_init)
                        uvmpagerops[lcv]->pgo_init();
        }
}

#ifdef PMAP_DIRECT
/*
 * uvm_pagermapdirect: map a single page via the pmap's direct segment
 *
 * this is an abuse of pmap_direct_process(), since the kva is being grabbed
 * and no processing is taking place, but for now..
 */

static int
uvm_pagermapdirect(void *kva, size_t sz, void *cookie)
{

        KASSERT(sz == PAGE_SIZE);
        *(vaddr_t *)cookie = (vaddr_t)kva;
        return 0;
}
#endif

/*
 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings
 *
 * we basically just map in a blank map entry to reserve the space in the
 * map and then use pmap_enter() to put the mappings in by hand.
 */

vaddr_t
uvm_pagermapin(struct vm_page **pps, int npages, int flags)
{
        vsize_t size;
        vaddr_t kva;
        vaddr_t cva;
        struct vm_page *pp;
        vm_prot_t prot;
        const bool pdaemon = uvm_lwp_is_pagedaemon(curlwp);
        const u_int first_color = VM_PGCOLOR(*pps);
        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(maphist,"(pps=%#jx, npages=%jd, first_color=%ju)",
                (uintptr_t)pps, npages, first_color, 0);

#ifdef PMAP_DIRECT
        /*
         * for a single page the direct mapped segment can be used.
         */

        if (npages == 1) {
                int error __diagused;
                KASSERT((pps[0]->flags & PG_BUSY) != 0);
                error = pmap_direct_process(VM_PAGE_TO_PHYS(pps[0]), 0,
                    PAGE_SIZE, uvm_pagermapdirect, &kva);
                KASSERT(error == 0);
                UVMHIST_LOG(maphist, "<- done, direct (KVA=%#jx)", kva,0,0,0);
                return kva;
        }
#endif

        /*
         * compute protection.  outgoing I/O only needs read
         * access to the page, whereas incoming needs read/write.
         */

        prot = VM_PROT_READ;
        if (flags & UVMPAGER_MAPIN_READ)
                prot |= VM_PROT_WRITE;

ReStart:
        size = ptoa(npages);
        kva = 0;                        /* let system choose VA */

        if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET,
            first_color, UVM_FLAG_COLORMATCH | UVM_FLAG_NOMERGE
            | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) {
                if (pdaemon) {
                        mutex_enter(&pager_map_wanted_lock);
                        if (emerginuse) {
                                UVM_UNLOCK_AND_WAIT(&emergva,
                                    &pager_map_wanted_lock, false,
                                    "emergva", 0);
                                goto ReStart;
                        }
                        emerginuse = true;
                        mutex_exit(&pager_map_wanted_lock);
                        kva = emergva + ptoa(first_color);
                        /* The shift implicitly truncates to PAGE_SIZE */
                        KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT));
                        goto enter;
                }
                if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) {
                        UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0);
                        return(0);
                }
                mutex_enter(&pager_map_wanted_lock);
                pager_map_wanted = true;
                UVMHIST_LOG(maphist, "  SLEEPING on pager_map",0,0,0,0);
                UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, false,
                    "pager_map", 0);
                goto ReStart;
        }

enter:
        /* got it */
        for (cva = kva; npages != 0; npages--, cva += PAGE_SIZE) {
                pp = *pps++;
                KASSERT(pp);
                // KASSERT(!((VM_PAGE_TO_PHYS(pp) ^ cva) & uvmexp.colormask));
                KASSERT(pp->flags & PG_BUSY);
                pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot, 0);
        }
        pmap_update(vm_map_pmap(pager_map));

        UVMHIST_LOG(maphist, "<- done (KVA=%#jx)", kva,0,0,0);
        return(kva);
}

/*
 * uvm_pagermapout: remove pager_map mapping
 *
 * we remove our mappings by hand and then remove the mapping (waking
 * up anyone wanting space).
 */

void
uvm_pagermapout(vaddr_t kva, int npages)
{
        vsize_t size = ptoa(npages);
        struct vm_map_entry *entries;
        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(maphist, " (kva=%#jx, npages=%jd)", kva, npages,0,0);

#ifdef PMAP_DIRECT
        /*
         * solitary pages are mapped directly.
         */

        if (npages == 1) {
                UVMHIST_LOG(maphist,"<- done, direct", 0,0,0,0);
                return;
        }
#endif

        /*
         * duplicate uvm_unmap, but add in pager_map_wanted handling.
         */

        pmap_kremove(kva, size);
        pmap_update(pmap_kernel());

        if ((kva & ~ptoa(uvmexp.colormask)) == emergva) {
                mutex_enter(&pager_map_wanted_lock);
                KASSERT(emerginuse);
                emerginuse = false;
                wakeup(&emergva);
                mutex_exit(&pager_map_wanted_lock);
                return;
        }

        vm_map_lock(pager_map);
        uvm_unmap_remove(pager_map, kva, kva + size, &entries, 0);
        mutex_enter(&pager_map_wanted_lock);
        if (pager_map_wanted) {
                pager_map_wanted = false;
                wakeup(pager_map);
        }
        mutex_exit(&pager_map_wanted_lock);
        vm_map_unlock(pager_map);
        if (entries)
                uvm_unmap_detach(entries, 0);
        UVMHIST_LOG(maphist,"<- done",0,0,0,0);
}

static int
uvm_page_swapslot(struct vm_page *pg)
{
        int swslot;

        KASSERT((pg->flags & PG_SWAPBACKED) != 0);
        if (pg->uobject != NULL) {
                swslot = uao_find_swslot(pg->uobject, pg->offset >> PAGE_SHIFT);
        } else {
                KASSERT(pg->uanon != NULL);
                swslot = pg->uanon->an_swslot;
        }
        return swslot;
}

void
uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error)
{
        struct uvm_object *uobj;
        struct vm_page *pg;
        krwlock_t *slock;
        int pageout_done;       /* number of PG_PAGEOUT pages processed */
        int swslot __unused;    /* used for VMSWAP */
        int i;
        bool swap;
        UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);

        swslot = 0;
        pageout_done = 0;
        slock = NULL;
        uobj = NULL;
        pg = pgs[0];
        swap = (pg->uanon != NULL && pg->uobject == NULL) ||
                (pg->flags & PG_AOBJ) != 0;
        if (!swap) {
                uobj = pg->uobject;
                slock = uobj->vmobjlock;
                rw_enter(slock, RW_WRITER);
        } else {
#if defined(VMSWAP)
                if (error) {
                        swslot = uvm_page_swapslot(pg);
                        KASSERT(swslot);
                }
#else /* defined(VMSWAP) */
                panic("%s: swap", __func__);
#endif /* defined(VMSWAP) */
        }
        for (i = 0; i < npages; i++) {
#if defined(VMSWAP)
                bool anon_disposed = false; /* XXX gcc */
#endif /* defined(VMSWAP) */

                pg = pgs[i];
                KASSERT(swap || pg->uobject == uobj);
                UVMHIST_LOG(ubchist, "pg %#jx", (uintptr_t)pg, 0,0,0);

#if defined(VMSWAP)
                /*
                 * for swap i/os, lock each page's object (or anon)
                 * individually since each page may need a different lock.
                 */

                if (swap) {
                        if (pg->uobject != NULL) {
                                slock = pg->uobject->vmobjlock;
                        } else {
                                slock = pg->uanon->an_lock;
                        }
                        rw_enter(slock, RW_WRITER);
                        anon_disposed = (pg->flags & PG_RELEASED) != 0;
                        KASSERT(!anon_disposed || pg->uobject != NULL ||
                            pg->uanon->an_ref == 0);
                }
#endif /* defined(VMSWAP) */

                if (write && uobj != NULL) {
                        KASSERT(uvm_obj_page_writeback_p(pg));
                        uvm_obj_page_clear_writeback(pg);
                }

                /*
                 * process errors.  for reads, just mark the page to be freed.
                 * for writes, if the error was ENOMEM, we assume this was
                 * a transient failure so we mark the page dirty so that
                 * we'll try to write it again later. for all other write
                 * errors for non swap backed pages, we assume the error
                 * is permanent, thus the data in the page is lost.  bummer.
                 *
                 * for swap backed pages, disassociate the swap slot and
                 * cancel the pageout attempt. mark the swap slot bad for
                 * errors except ENOMEM.
                 */

                if (error) {
                        if (!write) {
                                pg->flags |= PG_RELEASED;
                                continue;
                        }
                        if (swap || error == ENOMEM) {
                                if (pg->flags & PG_PAGEOUT) {
                                        pg->flags &= ~PG_PAGEOUT;
                                        pageout_done++;
                                }
                                uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
                                uvm_pagelock(pg);
                                uvm_pageactivate(pg);
                                uvm_pageunlock(pg);
                        }

#if defined(VMSWAP)
                        if (swap) {
                                /*
                                 * disassociate the swap slot from the page.
                                 * these slots will be either freed or
                                 * marked bad later in this function.
                                 */
                                if (pg->uobject != NULL) {
                                        int oldslot __diagused;
                                        oldslot = uao_set_swslot(pg->uobject,
                                                pg->offset >> PAGE_SHIFT, 0);
                                        KASSERT(oldslot == swslot + i);
                                } else {
                                        KASSERT(pg->uanon->an_swslot ==
                                                swslot + i);
                                        pg->uanon->an_swslot = 0;
                                }
                        }
#endif /* defined(VMSWAP) */
                }

                /*
                 * if the page is PG_FAKE, this must have been a read to
                 * initialize the page.  clear PG_FAKE and activate the page.
                 */

                if (pg->flags & PG_FAKE) {
                        KASSERT(!write);
                        pg->flags &= ~PG_FAKE;
#if defined(READAHEAD_STATS)
                        pg->flags |= PG_READAHEAD;
                        uvm_ra_total.ev_count++;
#endif /* defined(READAHEAD_STATS) */
                        KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN);
                        uvm_pagelock(pg);
                        uvm_pageenqueue(pg);
                        uvm_pageunlock(pg);
                }

#if defined(VMSWAP)
                /*
                 * for swap pages, unlock everything for this page now.
                 */

                if (swap) {
                        if (pg->uobject == NULL && anon_disposed) {
                                uvm_anon_release(pg->uanon);
                        } else {
                                uvm_page_unbusy(&pg, 1);
                                rw_exit(slock);
                        }
                }
#endif /* defined(VMSWAP) */
        }
        if (pageout_done != 0) {
                uvm_pageout_done(pageout_done);
        }
        if (!swap) {
                uvm_page_unbusy(pgs, npages);
                rw_exit(slock);
        } else {
#if defined(VMSWAP)
                KASSERT(write);

                /*
                 * these pages are now only in swap.
                 *
                 * note about swpgonly:
                 *
                 * if no errors, we increase swpgonly.
                 *
                 * on error which is not ENOMEM, we increase swpgonly.
                 * and then uvm_swap_markbad() decreases it by the same
                 * amount.
                 */
                if (error != ENOMEM) {
                        atomic_add_int(&uvmexp.swpgonly, npages);
                }
                if (error) {
                        if (error != ENOMEM)
                                uvm_swap_markbad(swslot, npages);
                        else
                                uvm_swap_free(swslot, npages);
                }
#endif /* defined(VMSWAP) */
        }
}

/*
 * uvm_aio_aiodone: do iodone processing for async i/os.
 * this should be called in thread context, not interrupt context.
 */
void
uvm_aio_aiodone(struct buf *bp)
{
        const int npages = bp->b_bufsize >> PAGE_SHIFT;
        struct vm_page *pgs[howmany(MAXPHYS, MIN_PAGE_SIZE)];
        int i, error;
        bool write;
        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(ubchist, "bp %#jx", (uintptr_t)bp, 0,0,0);

        KASSERT(bp->b_bufsize <= MAXPHYS);
        KASSERT(npages <= __arraycount(pgs));

        error = bp->b_error;
        write = (bp->b_flags & B_READ) == 0;

        for (i = 0; i < npages; i++) {
                pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT));
                UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i,
                    (uintptr_t)pgs[i], 0, 0);
        }

#if defined(VMSWAP)
        if (__predict_false(error != 0) &&
            ((pgs[0]->flags & PG_SWAPBACKED) != 0)) {
                int swslot = uvm_page_swapslot(pgs[0]);

                KASSERT(swslot > 0);
                uvm_swap_decrypt_pages(swslot, bp->b_data, npages);
        }
#endif

        uvm_pagermapout((vaddr_t)bp->b_data, npages);

        uvm_aio_aiodone_pages(pgs, npages, write, error);

        if (write && (bp->b_cflags & BC_AGE) != 0) {
                mutex_enter(bp->b_objlock);
                vwakeup(bp);
                mutex_exit(bp->b_objlock);
        }
        putiobuf(bp);
}

/*
 * uvm_pageratop: convert KVAs in the pager map back to their page
 * structures.
 */

struct vm_page *
uvm_pageratop(vaddr_t kva)
{
        struct vm_page *pg;
        paddr_t pa;
        bool rv __diagused;

        rv = pmap_extract(pmap_kernel(), kva, &pa);
        KASSERT(rv);
        pg = PHYS_TO_VM_PAGE(pa);
        KASSERT(pg != NULL);
        return (pg);
}