sys/uvm/uvm_bio.c

root/sys/uvm/uvm_bio.c
/*      $NetBSD: uvm_bio.c,v 1.128 2023/04/09 09:00:56 riastradh Exp $  */

/*
 * Copyright (c) 1998 Chuck Silvers.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 */

/*
 * uvm_bio.c: buffered i/o object mapping cache
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.128 2023/04/09 09:00:56 riastradh Exp $");

#include "opt_uvmhist.h"
#include "opt_ubc.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kmem.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/vnode.h>
#include <sys/bitops.h>         /* for ilog2() */

#include <uvm/uvm.h>
#include <uvm/uvm_pdpolicy.h>

#ifdef PMAP_DIRECT
#  define UBC_USE_PMAP_DIRECT
#endif

/*
 * local functions
 */

static int      ubc_fault(struct uvm_faultinfo *, vaddr_t, struct vm_page **,
                          int, int, vm_prot_t, int);
static struct ubc_map *ubc_find_mapping(struct uvm_object *, voff_t);
static int      ubchash_stats(struct hashstat_sysctl *hs, bool fill);
#ifdef UBC_USE_PMAP_DIRECT
static int __noinline ubc_uiomove_direct(struct uvm_object *, struct uio *, vsize_t,
                          int, int);
static void __noinline ubc_zerorange_direct(struct uvm_object *, off_t, size_t, int);

/* XXX disabled by default until the kinks are worked out. */
bool ubc_direct = false;
#endif

/*
 * local data structures
 */

#define UBC_HASH(uobj, offset)                                          \
        (((((u_long)(uobj)) >> 8) + (((u_long)(offset)) >> PAGE_SHIFT)) & \
                                ubc_object.hashmask)

#define UBC_QUEUE(offset)                                               \
        (&ubc_object.inactive[(((u_long)(offset)) >> ubc_winshift) &    \
                             (UBC_NQUEUES - 1)])

#define UBC_UMAP_ADDR(u)                                                \
        (vaddr_t)(ubc_object.kva + (((u) - ubc_object.umap) << ubc_winshift))


#define UMAP_PAGES_LOCKED       0x0001
#define UMAP_MAPPING_CACHED     0x0002

struct ubc_map {
        struct uvm_object *     uobj;           /* mapped object */
        voff_t                  offset;         /* offset into uobj */
        voff_t                  writeoff;       /* write offset */
        vsize_t                 writelen;       /* write len */
        int                     refcount;       /* refcount on mapping */
        int                     flags;          /* extra state */
        int                     advice;

        LIST_ENTRY(ubc_map)     hash;           /* hash table */
        TAILQ_ENTRY(ubc_map)    inactive;       /* inactive queue */
        LIST_ENTRY(ubc_map)     list;           /* per-object list */
};

TAILQ_HEAD(ubc_inactive_head, ubc_map);
static struct ubc_object {
        struct uvm_object uobj;         /* glue for uvm_map() */
        char *kva;                      /* where ubc_object is mapped */
        struct ubc_map *umap;           /* array of ubc_map's */

        LIST_HEAD(, ubc_map) *hash;     /* hashtable for cached ubc_map's */
        u_long hashmask;                /* mask for hashtable */

        struct ubc_inactive_head *inactive;
                                        /* inactive queues for ubc_map's */
} ubc_object;

const struct uvm_pagerops ubc_pager = {
        .pgo_fault = ubc_fault,
        /* ... rest are NULL */
};

/* Use value at least as big as maximum page size supported by architecture */
#define UBC_MAX_WINSHIFT        \
    ((1 << UBC_WINSHIFT) > MAX_PAGE_SIZE ? UBC_WINSHIFT : ilog2(MAX_PAGE_SIZE))

int ubc_nwins = UBC_NWINS;
const int ubc_winshift = UBC_MAX_WINSHIFT;
const int ubc_winsize = 1 << UBC_MAX_WINSHIFT;
#if defined(PMAP_PREFER)
int ubc_nqueues;
#define UBC_NQUEUES ubc_nqueues
#else
#define UBC_NQUEUES 1
#endif

#if defined(UBC_STATS)

#define UBC_EVCNT_DEFINE(name) \
struct evcnt ubc_evcnt_##name = \
EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "ubc", #name); \
EVCNT_ATTACH_STATIC(ubc_evcnt_##name);
#define UBC_EVCNT_INCR(name) ubc_evcnt_##name.ev_count++

#else /* defined(UBC_STATS) */

#define UBC_EVCNT_DEFINE(name)  /* nothing */
#define UBC_EVCNT_INCR(name)    /* nothing */

#endif /* defined(UBC_STATS) */

UBC_EVCNT_DEFINE(wincachehit)
UBC_EVCNT_DEFINE(wincachemiss)
UBC_EVCNT_DEFINE(faultbusy)

/*
 * ubc_init
 *
 * init pager private data structures.
 */

void
ubc_init(void)
{
        /*
         * Make sure ubc_winshift is sane.
         */
        KASSERT(ubc_winshift >= PAGE_SHIFT);

        /*
         * init ubc_object.
         * alloc and init ubc_map's.
         * init inactive queues.
         * alloc and init hashtable.
         * map in ubc_object.
         */

        uvm_obj_init(&ubc_object.uobj, &ubc_pager, true, UVM_OBJ_KERN);

        ubc_object.umap = kmem_zalloc(ubc_nwins * sizeof(struct ubc_map),
            KM_SLEEP);
        if (ubc_object.umap == NULL)
                panic("ubc_init: failed to allocate ubc_map");

        vaddr_t va = (vaddr_t)1L;
#ifdef PMAP_PREFER
        PMAP_PREFER(0, &va, 0, 0);      /* kernel is never topdown */
        ubc_nqueues = va >> ubc_winshift;
        if (ubc_nqueues == 0) {
                ubc_nqueues = 1;
        }
#endif
        ubc_object.inactive = kmem_alloc(UBC_NQUEUES *
            sizeof(struct ubc_inactive_head), KM_SLEEP);
        for (int i = 0; i < UBC_NQUEUES; i++) {
                TAILQ_INIT(&ubc_object.inactive[i]);
        }
        for (int i = 0; i < ubc_nwins; i++) {
                struct ubc_map *umap;
                umap = &ubc_object.umap[i];
                TAILQ_INSERT_TAIL(&ubc_object.inactive[i & (UBC_NQUEUES - 1)],
                                  umap, inactive);
        }

        ubc_object.hash = hashinit(ubc_nwins, HASH_LIST, true,
            &ubc_object.hashmask);
        for (int i = 0; i <= ubc_object.hashmask; i++) {
                LIST_INIT(&ubc_object.hash[i]);
        }

        if (uvm_map(kernel_map, (vaddr_t *)&ubc_object.kva,
                    ubc_nwins << ubc_winshift, &ubc_object.uobj, 0, (vsize_t)va,
                    UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE,
                                UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != 0) {
                panic("ubc_init: failed to map ubc_object");
        }

        hashstat_register("ubchash", ubchash_stats);
}

void
ubchist_init(void)
{

        UVMHIST_INIT(ubchist, 300);
}

/*
 * ubc_fault_page: helper of ubc_fault to handle a single page.
 *
 * => Caller has UVM object locked.
 * => Caller will perform pmap_update().
 */

static inline int
ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
    struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va)
{
        vm_prot_t mask;
        int error;
        bool rdonly;

        KASSERT(rw_write_held(pg->uobject->vmobjlock));

        KASSERT((pg->flags & PG_FAKE) == 0);
        if (pg->flags & PG_RELEASED) {
                uvm_pagefree(pg);
                return 0;
        }
        if (pg->loan_count != 0) {

                /*
                 * Avoid unneeded loan break, if possible.
                 */

                if ((access_type & VM_PROT_WRITE) == 0) {
                        prot &= ~VM_PROT_WRITE;
                }
                if (prot & VM_PROT_WRITE) {
                        struct vm_page *newpg;

                        newpg = uvm_loanbreak(pg);
                        if (newpg == NULL) {
                                uvm_page_unbusy(&pg, 1);
                                return ENOMEM;
                        }
                        pg = newpg;
                }
        }

        /*
         * Note that a page whose backing store is partially allocated
         * is marked as PG_RDONLY.
         *
         * it's a responsibility of ubc_alloc's caller to allocate backing
         * blocks before writing to the window.
         */

        KASSERT((pg->flags & PG_RDONLY) == 0 ||
            (access_type & VM_PROT_WRITE) == 0 ||
            pg->offset < umap->writeoff ||
            pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen);

        rdonly = uvm_pagereadonly_p(pg);
        mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL;

        error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg),
            prot & mask, PMAP_CANFAIL | (access_type & mask));

        uvm_pagelock(pg);
        uvm_pageactivate(pg);
        uvm_pagewakeup(pg);
        uvm_pageunlock(pg);
        pg->flags &= ~PG_BUSY;
        UVM_PAGE_OWN(pg, NULL);

        return error;
}

/*
 * ubc_fault: fault routine for ubc mapping
 */

static int
ubc_fault(struct uvm_faultinfo *ufi, vaddr_t ign1, struct vm_page **ign2,
    int ign3, int ign4, vm_prot_t access_type, int flags)
{
        struct uvm_object *uobj;
        struct ubc_map *umap;
        vaddr_t va, eva, ubc_offset, slot_offset;
        struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
        int i, error, npages;
        vm_prot_t prot;

        UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);

        /*
         * no need to try with PGO_LOCKED...
         * we don't need to have the map locked since we know that
         * no one will mess with it until our reference is released.
         */

        if (flags & PGO_LOCKED) {
                uvmfault_unlockall(ufi, NULL, &ubc_object.uobj);
                flags &= ~PGO_LOCKED;
        }

        va = ufi->orig_rvaddr;
        ubc_offset = va - (vaddr_t)ubc_object.kva;
        umap = &ubc_object.umap[ubc_offset >> ubc_winshift];
        KASSERT(umap->refcount != 0);
        KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0);
        slot_offset = ubc_offset & (ubc_winsize - 1);

        /*
         * some platforms cannot write to individual bytes atomically, so
         * software has to do read/modify/write of larger quantities instead.
         * this means that the access_type for "write" operations
         * can be VM_PROT_READ, which confuses us mightily.
         *
         * deal with this by resetting access_type based on the info
         * that ubc_alloc() stores for us.
         */

        access_type = umap->writelen ? VM_PROT_WRITE : VM_PROT_READ;
        UVMHIST_LOG(ubchist, "va %#jx ubc_offset %#jx access_type %jd",
            va, ubc_offset, access_type, 0);

        if ((access_type & VM_PROT_WRITE) != 0) {
#ifndef PRIxOFF         /* XXX */
#define PRIxOFF "jx"    /* XXX */
#endif                  /* XXX */
                KASSERTMSG((trunc_page(umap->writeoff) <= slot_offset),
                    "out of range write: slot=%#"PRIxVSIZE" off=%#"PRIxOFF,
                    slot_offset, (intmax_t)umap->writeoff);
                KASSERTMSG((slot_offset < umap->writeoff + umap->writelen),
                    "out of range write: slot=%#"PRIxVADDR
                        " off=%#"PRIxOFF" len=%#"PRIxVSIZE,
                    slot_offset, (intmax_t)umap->writeoff, umap->writelen);
        }

        /* no umap locking needed since we have a ref on the umap */
        uobj = umap->uobj;

        if ((access_type & VM_PROT_WRITE) == 0) {
                npages = (ubc_winsize - slot_offset) >> PAGE_SHIFT;
        } else {
                npages = (round_page(umap->offset + umap->writeoff +
                    umap->writelen) - (umap->offset + slot_offset))
                    >> PAGE_SHIFT;
                flags |= PGO_PASTEOF;
        }

again:
        memset(pgs, 0, sizeof (pgs));
        rw_enter(uobj->vmobjlock, RW_WRITER);

        UVMHIST_LOG(ubchist, "slot_offset %#jx writeoff %#jx writelen %#jx ",
            slot_offset, umap->writeoff, umap->writelen, 0);
        UVMHIST_LOG(ubchist, "getpages uobj %#jx offset %#jx npages %jd",
            (uintptr_t)uobj, umap->offset + slot_offset, npages, 0);

        error = (*uobj->pgops->pgo_get)(uobj, umap->offset + slot_offset, pgs,
            &npages, 0, access_type, umap->advice, flags | PGO_NOBLOCKALLOC |
            PGO_NOTIMESTAMP);
        UVMHIST_LOG(ubchist, "getpages error %jd npages %jd", error, npages, 0,
            0);

        if (error == EAGAIN) {
                kpause("ubc_fault", false, hz >> 2, NULL);
                goto again;
        }
        if (error) {
                return error;
        }

        /*
         * For virtually-indexed, virtually-tagged caches we should avoid
         * creating writable mappings when we do not absolutely need them,
         * since the "compatible alias" trick does not work on such caches.
         * Otherwise, we can always map the pages writable.
         */

#ifdef PMAP_CACHE_VIVT
        prot = VM_PROT_READ | access_type;
#else
        prot = VM_PROT_READ | VM_PROT_WRITE;
#endif

        va = ufi->orig_rvaddr;
        eva = ufi->orig_rvaddr + (npages << PAGE_SHIFT);

        UVMHIST_LOG(ubchist, "va %#jx eva %#jx", va, eva, 0, 0);

        /*
         * Note: normally all returned pages would have the same UVM object.
         * However, layered file-systems and e.g. tmpfs, may return pages
         * which belong to underlying UVM object.  In such case, lock is
         * shared amongst the objects.
         */
        rw_enter(uobj->vmobjlock, RW_WRITER);
        for (i = 0; va < eva; i++, va += PAGE_SIZE) {
                struct vm_page *pg;

                UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, (uintptr_t)pgs[i],
                    0, 0);
                pg = pgs[i];

                if (pg == NULL || pg == PGO_DONTCARE) {
                        continue;
                }
                KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock);
                error = ubc_fault_page(ufi, umap, pg, prot, access_type, va);
                if (error) {
                        /*
                         * Flush (there might be pages entered), drop the lock,
                         * and perform uvm_wait().  Note: page will re-fault.
                         */
                        pmap_update(ufi->orig_map->pmap);
                        rw_exit(uobj->vmobjlock);
                        uvm_wait("ubc_fault");
                        rw_enter(uobj->vmobjlock, RW_WRITER);
                }
        }
        /* Must make VA visible before the unlock. */
        pmap_update(ufi->orig_map->pmap);
        rw_exit(uobj->vmobjlock);

        return 0;
}

/*
 * local functions
 */

static struct ubc_map *
ubc_find_mapping(struct uvm_object *uobj, voff_t offset)
{
        struct ubc_map *umap;

        LIST_FOREACH(umap, &ubc_object.hash[UBC_HASH(uobj, offset)], hash) {
                if (umap->uobj == uobj && umap->offset == offset) {
                        return umap;
                }
        }
        return NULL;
}


/*
 * ubc interface functions
 */

/*
 * ubc_alloc:  allocate a file mapping window
 */

static void * __noinline
ubc_alloc(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice,
    int flags, struct vm_page **pgs, int *npagesp)
{
        vaddr_t slot_offset, va;
        struct ubc_map *umap;
        voff_t umap_offset;
        int error;
        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(ubchist, "uobj %#jx offset %#jx len %#jx",
            (uintptr_t)uobj, offset, *lenp, 0);

        KASSERT(*lenp > 0);
        umap_offset = (offset & ~((voff_t)ubc_winsize - 1));
        slot_offset = (vaddr_t)(offset & ((voff_t)ubc_winsize - 1));
        *lenp = MIN(*lenp, ubc_winsize - slot_offset);
        KASSERT(*lenp > 0);

        rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
again:
        /*
         * The UVM object is already referenced.
         * Lock order: UBC object -> ubc_map::uobj.
         */
        umap = ubc_find_mapping(uobj, umap_offset);
        if (umap == NULL) {
                struct uvm_object *oobj;

                UBC_EVCNT_INCR(wincachemiss);
                umap = TAILQ_FIRST(UBC_QUEUE(offset));
                if (umap == NULL) {
                        rw_exit(ubc_object.uobj.vmobjlock);
                        kpause("ubc_alloc", false, hz >> 2, NULL);
                        rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
                        goto again;
                }

                va = UBC_UMAP_ADDR(umap);
                oobj = umap->uobj;

                /*
                 * Remove from old hash (if any), add to new hash.
                 */

                if (oobj != NULL) {
                        /*
                         * Mapping must be removed before the list entry,
                         * since there is a race with ubc_purge().
                         */
                        if (umap->flags & UMAP_MAPPING_CACHED) {
                                umap->flags &= ~UMAP_MAPPING_CACHED;
                                rw_enter(oobj->vmobjlock, RW_WRITER);
                                pmap_remove(pmap_kernel(), va,
                                    va + ubc_winsize);
                                pmap_update(pmap_kernel());
                                rw_exit(oobj->vmobjlock);
                        }
                        LIST_REMOVE(umap, hash);
                        LIST_REMOVE(umap, list);
                } else {
                        KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0);
                }
                umap->uobj = uobj;
                umap->offset = umap_offset;
                LIST_INSERT_HEAD(&ubc_object.hash[UBC_HASH(uobj, umap_offset)],
                    umap, hash);
                LIST_INSERT_HEAD(&uobj->uo_ubc, umap, list);
        } else {
                UBC_EVCNT_INCR(wincachehit);
                va = UBC_UMAP_ADDR(umap);
        }

        if (umap->refcount == 0) {
                TAILQ_REMOVE(UBC_QUEUE(offset), umap, inactive);
        }

        if (flags & UBC_WRITE) {
                KASSERTMSG(umap->writeoff == 0,
                    "ubc_alloc: concurrent writes to uobj %p", uobj);
                KASSERTMSG(umap->writelen == 0,
                    "ubc_alloc: concurrent writes to uobj %p", uobj);
                umap->writeoff = slot_offset;
                umap->writelen = *lenp;
        }

        umap->refcount++;
        umap->advice = advice;
        rw_exit(ubc_object.uobj.vmobjlock);
        UVMHIST_LOG(ubchist, "umap %#jx refs %jd va %#jx flags %#jx",
            (uintptr_t)umap, umap->refcount, (uintptr_t)va, flags);

        if (flags & UBC_FAULTBUSY) {
                int npages = (*lenp + (offset & (PAGE_SIZE - 1)) +
                    PAGE_SIZE - 1) >> PAGE_SHIFT;
                int gpflags =
                    PGO_SYNCIO|PGO_OVERWRITE|PGO_PASTEOF|PGO_NOBLOCKALLOC|
                    PGO_NOTIMESTAMP;
                int i;
                KDASSERT(flags & UBC_WRITE);
                KASSERT(npages <= *npagesp);
                KASSERT(umap->refcount == 1);

                UBC_EVCNT_INCR(faultbusy);
again_faultbusy:
                rw_enter(uobj->vmobjlock, RW_WRITER);
                if (umap->flags & UMAP_MAPPING_CACHED) {
                        umap->flags &= ~UMAP_MAPPING_CACHED;
                        pmap_remove(pmap_kernel(), va, va + ubc_winsize);
                }
                memset(pgs, 0, *npagesp * sizeof(pgs[0]));

                error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs,
                    &npages, 0, VM_PROT_READ | VM_PROT_WRITE, advice, gpflags);
                UVMHIST_LOG(ubchist, "faultbusy getpages %jd", error, 0, 0, 0);
                if (error) {
                        /*
                         * Flush: the mapping above might have been removed.
                         */
                        pmap_update(pmap_kernel());
                        goto out;
                }
                for (i = 0; i < npages; i++) {
                        struct vm_page *pg = pgs[i];

                        KASSERT(pg->uobject == uobj);
                        if (pg->loan_count != 0) {
                                rw_enter(uobj->vmobjlock, RW_WRITER);
                                if (pg->loan_count != 0) {
                                        pg = uvm_loanbreak(pg);
                                }
                                if (pg == NULL) {
                                        pmap_kremove(va, ubc_winsize);
                                        pmap_update(pmap_kernel());
                                        uvm_page_unbusy(pgs, npages);
                                        rw_exit(uobj->vmobjlock);
                                        uvm_wait("ubc_alloc");
                                        goto again_faultbusy;
                                }
                                rw_exit(uobj->vmobjlock);
                                pgs[i] = pg;
                        }
                        pmap_kenter_pa(
                            va + trunc_page(slot_offset) + (i << PAGE_SHIFT),
                            VM_PAGE_TO_PHYS(pg),
                            VM_PROT_READ | VM_PROT_WRITE, 0);
                }
                pmap_update(pmap_kernel());
                umap->flags |= UMAP_PAGES_LOCKED;
                *npagesp = npages;
        } else {
                KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0);
        }

out:
        return (void *)(va + slot_offset);
}

/*
 * ubc_release:  free a file mapping window.
 */

static void __noinline
ubc_release(void *va, int flags, struct vm_page **pgs, int npages)
{
        struct ubc_map *umap;
        struct uvm_object *uobj;
        vaddr_t umapva;
        bool unmapped;
        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(ubchist, "va %#jx", (uintptr_t)va, 0, 0, 0);

        umap = &ubc_object.umap[((char *)va - ubc_object.kva) >> ubc_winshift];
        umapva = UBC_UMAP_ADDR(umap);
        uobj = umap->uobj;
        KASSERT(uobj != NULL);

        if (umap->flags & UMAP_PAGES_LOCKED) {
                const voff_t endoff = umap->writeoff + umap->writelen;
                const voff_t zerolen = round_page(endoff) - endoff;

                KASSERT(npages == (round_page(endoff) -
                    trunc_page(umap->writeoff)) >> PAGE_SHIFT);
                KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0);
                if (zerolen) {
                        memset((char *)umapva + endoff, 0, zerolen);
                }
                umap->flags &= ~UMAP_PAGES_LOCKED;
                rw_enter(uobj->vmobjlock, RW_WRITER);
                for (u_int i = 0; i < npages; i++) {
                        struct vm_page *pg = pgs[i];
#ifdef DIAGNOSTIC
                        paddr_t pa;
                        bool rv;
                        rv = pmap_extract(pmap_kernel(), umapva +
                            umap->writeoff + (i << PAGE_SHIFT), &pa);
                        KASSERT(rv);
                        KASSERT(PHYS_TO_VM_PAGE(pa) == pg);
#endif
                        pg->flags &= ~PG_FAKE;
                        KASSERTMSG(uvm_pagegetdirty(pg) ==
                            UVM_PAGE_STATUS_DIRTY,
                            "page %p not dirty", pg);
                        KASSERT(pg->loan_count == 0);
                        if (uvmpdpol_pageactivate_p(pg)) {
                                uvm_pagelock(pg);
                                uvm_pageactivate(pg);
                                uvm_pageunlock(pg);
                        }
                }
                pmap_kremove(umapva, ubc_winsize);
                pmap_update(pmap_kernel());
                uvm_page_unbusy(pgs, npages);
                rw_exit(uobj->vmobjlock);
                unmapped = true;
        } else {
                unmapped = false;
        }

        rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
        umap->writeoff = 0;
        umap->writelen = 0;
        umap->refcount--;
        if (umap->refcount == 0) {
                if (flags & UBC_UNMAP) {
                        /*
                         * Invalidate any cached mappings if requested.
                         * This is typically used to avoid leaving
                         * incompatible cache aliases around indefinitely.
                         */
                        rw_enter(uobj->vmobjlock, RW_WRITER);
                        pmap_remove(pmap_kernel(), umapva,
                                    umapva + ubc_winsize);
                        pmap_update(pmap_kernel());
                        rw_exit(uobj->vmobjlock);

                        umap->flags &= ~UMAP_MAPPING_CACHED;
                        LIST_REMOVE(umap, hash);
                        LIST_REMOVE(umap, list);
                        umap->uobj = NULL;
                        TAILQ_INSERT_HEAD(UBC_QUEUE(umap->offset), umap,
                            inactive);
                } else {
                        if (!unmapped) {
                                umap->flags |= UMAP_MAPPING_CACHED;
                        }
                        TAILQ_INSERT_TAIL(UBC_QUEUE(umap->offset), umap,
                            inactive);
                }
        }
        UVMHIST_LOG(ubchist, "umap %#jx refs %jd", (uintptr_t)umap,
            umap->refcount, 0, 0);
        rw_exit(ubc_object.uobj.vmobjlock);
}

/*
 * ubc_uiomove: move data to/from an object.
 */

int
ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice,
    int flags)
{
        const bool overwrite = (flags & UBC_FAULTBUSY) != 0;
        struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
        voff_t off;
        int error, npages;

        KASSERT(todo <= uio->uio_resid);
        KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) ||
            ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ));

#ifdef UBC_USE_PMAP_DIRECT
        /*
         * during direct access pages need to be held busy to prevent them
         * changing identity, and therefore if we read or write an object
         * into a mapped view of same we could deadlock while faulting.
         *
         * avoid the problem by disallowing direct access if the object
         * might be visible somewhere via mmap().
         *
         * XXX concurrent reads cause thundering herd issues with PG_BUSY.
         * In the future enable by default for writes or if ncpu<=2, and
         * make the toggle override that.
         */
        if ((ubc_direct && (flags & UBC_ISMAPPED) == 0) ||
            (flags & UBC_FAULTBUSY) != 0) {
                return ubc_uiomove_direct(uobj, uio, todo, advice, flags);
        }
#endif

        off = uio->uio_offset;
        error = 0;
        while (todo > 0) {
                vsize_t bytelen = todo;
                void *win;

                npages = __arraycount(pgs);
                win = ubc_alloc(uobj, off, &bytelen, advice, flags, pgs,
                    &npages);
                if (error == 0) {
                        error = uiomove(win, bytelen, uio);
                }
                if (error != 0 && overwrite) {
                        /*
                         * if we haven't initialized the pages yet,
                         * do it now.  it's safe to use memset here
                         * because we just mapped the pages above.
                         */
                        memset(win, 0, bytelen);
                }
                ubc_release(win, flags, pgs, npages);
                off += bytelen;
                todo -= bytelen;
                if (error != 0 && (flags & UBC_PARTIALOK) != 0) {
                        break;
                }
        }

        return error;
}

/*
 * ubc_zerorange: set a range of bytes in an object to zero.
 */

void
ubc_zerorange(struct uvm_object *uobj, off_t off, size_t len, int flags)
{
        struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
        int npages;

#ifdef UBC_USE_PMAP_DIRECT
        if (ubc_direct || (flags & UBC_FAULTBUSY) != 0) {
                ubc_zerorange_direct(uobj, off, len, flags);
                return;
        }
#endif

        /*
         * XXXUBC invent kzero() and use it
         */

        while (len) {
                void *win;
                vsize_t bytelen = len;

                npages = __arraycount(pgs);
                win = ubc_alloc(uobj, off, &bytelen, UVM_ADV_NORMAL, UBC_WRITE,
                    pgs, &npages);
                memset(win, 0, bytelen);
                ubc_release(win, flags, pgs, npages);

                off += bytelen;
                len -= bytelen;
        }
}

#ifdef UBC_USE_PMAP_DIRECT
/* Copy data using direct map */

/*
 * ubc_alloc_direct:  allocate a file mapping window using direct map
 */
static int __noinline
ubc_alloc_direct(struct uvm_object *uobj, voff_t offset, vsize_t *lenp,
    int advice, int flags, struct vm_page **pgs, int *npages)
{
        voff_t pgoff;
        int error;
        int gpflags = flags | PGO_NOTIMESTAMP | PGO_SYNCIO;
        int access_type = VM_PROT_READ;
        UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);

        if (flags & UBC_WRITE) {
                if (flags & UBC_FAULTBUSY)
                        gpflags |= PGO_OVERWRITE | PGO_NOBLOCKALLOC;
#if 0
                KASSERT(!UVM_OBJ_NEEDS_WRITEFAULT(uobj));
#endif

                /*
                 * Tell genfs_getpages() we already have the journal lock,
                 * allow allocation past current EOF.
                 */
                gpflags |= PGO_JOURNALLOCKED | PGO_PASTEOF;
                access_type |= VM_PROT_WRITE;
        } else {
                /* Don't need the empty blocks allocated, PG_RDONLY is okay */
                gpflags |= PGO_NOBLOCKALLOC;
        }

        pgoff = (offset & PAGE_MASK);
        *lenp = MIN(*lenp, ubc_winsize - pgoff);

again:
        *npages = (*lenp + pgoff + PAGE_SIZE - 1) >> PAGE_SHIFT;
        KASSERT((*npages * PAGE_SIZE) <= ubc_winsize);
        KASSERT(*lenp + pgoff <= ubc_winsize);
        memset(pgs, 0, *npages * sizeof(pgs[0]));

        rw_enter(uobj->vmobjlock, RW_WRITER);
        error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs,
            npages, 0, access_type, advice, gpflags);
        UVMHIST_LOG(ubchist, "alloc_direct getpages %jd", error, 0, 0, 0);
        if (error) {
                if (error == EAGAIN) {
                        kpause("ubc_alloc_directg", false, hz >> 2, NULL);
                        goto again;
                }
                return error;
        }

        rw_enter(uobj->vmobjlock, RW_WRITER);
        for (int i = 0; i < *npages; i++) {
                struct vm_page *pg = pgs[i];

                KASSERT(pg != NULL);
                KASSERT(pg != PGO_DONTCARE);
                KASSERT((pg->flags & PG_FAKE) == 0 || (gpflags & PGO_OVERWRITE));
                KASSERT(pg->uobject->vmobjlock == uobj->vmobjlock);

                /* Avoid breaking loan if possible, only do it on write */
                if ((flags & UBC_WRITE) && pg->loan_count != 0) {
                        pg = uvm_loanbreak(pg);
                        if (pg == NULL) {
                                uvm_page_unbusy(pgs, *npages);
                                rw_exit(uobj->vmobjlock);
                                uvm_wait("ubc_alloc_directl");
                                goto again;
                        }
                        pgs[i] = pg;
                }

                /* Page must be writable by now */
                KASSERT((pg->flags & PG_RDONLY) == 0 || (flags & UBC_WRITE) == 0);

                /*
                 * XXX For aobj pages.  No managed mapping - mark the page
                 * dirty.
                 */
                if ((flags & UBC_WRITE) != 0) {
                        uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
                }
        }
        rw_exit(uobj->vmobjlock);

        return 0;
}

static void __noinline
ubc_direct_release(struct uvm_object *uobj,
        int flags, struct vm_page **pgs, int npages)
{
        rw_enter(uobj->vmobjlock, RW_WRITER);
        for (int i = 0; i < npages; i++) {
                struct vm_page *pg = pgs[i];

                pg->flags &= ~PG_BUSY;
                UVM_PAGE_OWN(pg, NULL);
                if (pg->flags & PG_RELEASED) {
                        pg->flags &= ~PG_RELEASED;
                        uvm_pagefree(pg);
                        continue;
                }

                if (uvm_pagewanted_p(pg) || uvmpdpol_pageactivate_p(pg)) {
                        uvm_pagelock(pg);
                        uvm_pageactivate(pg);
                        uvm_pagewakeup(pg);
                        uvm_pageunlock(pg);
                }

                /* Page was changed, no longer fake and neither clean. */
                if (flags & UBC_WRITE) {
                        KASSERTMSG(uvm_pagegetdirty(pg) ==
                            UVM_PAGE_STATUS_DIRTY,
                            "page %p not dirty", pg);
                        pg->flags &= ~PG_FAKE;
                }
        }
        rw_exit(uobj->vmobjlock);
}

static int
ubc_uiomove_process(void *win, size_t len, void *arg)
{
        struct uio *uio = (struct uio *)arg;

        return uiomove(win, len, uio);
}

static int
ubc_zerorange_process(void *win, size_t len, void *arg)
{
        memset(win, 0, len);
        return 0;
}

static int __noinline
ubc_uiomove_direct(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice,
    int flags)
{
        const bool overwrite = (flags & UBC_FAULTBUSY) != 0;
        voff_t off;
        int error, npages;
        struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];

        KASSERT(todo <= uio->uio_resid);
        KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) ||
            ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ));

        off = uio->uio_offset;
        error = 0;
        while (todo > 0) {
                vsize_t bytelen = todo;

                error = ubc_alloc_direct(uobj, off, &bytelen, advice, flags,
                    pgs, &npages);
                if (error != 0) {
                        /* can't do anything, failed to get the pages */
                        break;
                }

                if (error == 0) {
                        error = uvm_direct_process(pgs, npages, off, bytelen,
                            ubc_uiomove_process, uio);
                }

                if (overwrite) {
                        voff_t endoff;

                        /*
                         * if we haven't initialized the pages yet due to an
                         * error above, do it now.
                         */
                        if (error != 0) {
                                (void) uvm_direct_process(pgs, npages, off,
                                    bytelen, ubc_zerorange_process, NULL);
                        }

                        off += bytelen;
                        todo -= bytelen;
                        endoff = off & (PAGE_SIZE - 1);

                        /*
                         * zero out the remaining portion of the final page
                         * (if any).
                         */
                        if (todo == 0 && endoff != 0) {
                                vsize_t zlen = PAGE_SIZE - endoff;
                                (void) uvm_direct_process(pgs + npages - 1, 1,
                                    off, zlen, ubc_zerorange_process, NULL);
                        }
                } else {
                        off += bytelen;
                        todo -= bytelen;
                }

                ubc_direct_release(uobj, flags, pgs, npages);

                if (error != 0 && ISSET(flags, UBC_PARTIALOK)) {
                        break;
                }
        }

        return error;
}

static void __noinline
ubc_zerorange_direct(struct uvm_object *uobj, off_t off, size_t todo, int flags)
{
        int error, npages;
        struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];

        flags |= UBC_WRITE;

        error = 0;
        while (todo > 0) {
                vsize_t bytelen = todo;

                error = ubc_alloc_direct(uobj, off, &bytelen, UVM_ADV_NORMAL,
                    flags, pgs, &npages);
                if (error != 0) {
                        /* can't do anything, failed to get the pages */
                        break;
                }

                error = uvm_direct_process(pgs, npages, off, bytelen,
                    ubc_zerorange_process, NULL);

                ubc_direct_release(uobj, flags, pgs, npages);

                off += bytelen;
                todo -= bytelen;
        }
}

#endif /* UBC_USE_PMAP_DIRECT */

/*
 * ubc_purge: disassociate ubc_map structures from an empty uvm_object.
 */

void
ubc_purge(struct uvm_object *uobj)
{
        struct ubc_map *umap;
        vaddr_t va;

        KASSERT(uobj->uo_npages == 0);

        /*
         * Safe to check without lock held, as ubc_alloc() removes
         * the mapping and list entry in the correct order.
         */
        if (__predict_true(LIST_EMPTY(&uobj->uo_ubc))) {
                return;
        }
        rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
        while ((umap = LIST_FIRST(&uobj->uo_ubc)) != NULL) {
                KASSERT(umap->refcount == 0);
                for (va = 0; va < ubc_winsize; va += PAGE_SIZE) {
                        KASSERT(!pmap_extract(pmap_kernel(),
                            va + UBC_UMAP_ADDR(umap), NULL));
                }
                LIST_REMOVE(umap, list);
                LIST_REMOVE(umap, hash);
                umap->flags &= ~UMAP_MAPPING_CACHED;
                umap->uobj = NULL;
        }
        rw_exit(ubc_object.uobj.vmobjlock);
}

static int
ubchash_stats(struct hashstat_sysctl *hs, bool fill)
{
        struct ubc_map *umap;
        uint64_t chain;

        strlcpy(hs->hash_name, "ubchash", sizeof(hs->hash_name));
        strlcpy(hs->hash_desc, "ubc object hash", sizeof(hs->hash_desc));
        if (!fill)
                return 0;

        hs->hash_size = ubc_object.hashmask + 1;

        for (size_t i = 0; i < hs->hash_size; i++) {
                chain = 0;
                rw_enter(ubc_object.uobj.vmobjlock, RW_READER);
                LIST_FOREACH(umap, &ubc_object.hash[i], hash) {
                        chain++;
                }
                rw_exit(ubc_object.uobj.vmobjlock);
                if (chain > 0) {
                        hs->hash_used++;
                        hs->hash_items += chain;
                        if (chain > hs->hash_maxchain)
                                hs->hash_maxchain = chain;
                }
                preempt_point();
        }

        return 0;
}