/* $NetBSD: udf_strat_rmw.c,v 1.32 2026/01/11 17:52:42 joe Exp $ */

/*
 * Copyright (c) 2006, 2008 Reinoud Zandijk
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <sys/cdefs.h>
#ifndef lint
__KERNEL_RCSID(0, "$NetBSD: udf_strat_rmw.c,v 1.32 2026/01/11 17:52:42 joe Exp $");
#endif /* not lint */


#if defined(_KERNEL_OPT)
#include "opt_compat_netbsd.h"
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/namei.h>
#include <sys/proc.h>
#include <sys/kernel.h>
#include <sys/vnode.h>
#include <miscfs/genfs/genfs_node.h>
#include <sys/mount.h>
#include <sys/buf.h>
#include <sys/file.h>
#include <sys/device.h>
#include <sys/disklabel.h>
#include <sys/ioctl.h>
#include <sys/malloc.h>
#include <sys/dirent.h>
#include <sys/stat.h>
#include <sys/conf.h>
#include <sys/kauth.h>
#include <sys/kthread.h>
#include <dev/clock_subr.h>

#include <fs/udf/ecma167-udf.h>
#include <fs/udf/udf_mount.h>

#include "udf.h"
#include "udf_subr.h"
#include "udf_bswap.h"


#define VTOI(vnode) ((struct udf_node *) (vnode)->v_data)
#define PRIV(ump) ((struct strat_private *) (ump)->strategy_private)
#define BTOE(buf) ((struct udf_eccline *) ((buf)->b_private))

/* --------------------------------------------------------------------- */

#define UDF_MAX_PACKET_SIZE     64                      /* DONT change this */

/* sheduler states */
#define UDF_SHED_WAITING        1                       /* waiting on timeout */
#define UDF_SHED_READING        2
#define UDF_SHED_WRITING        3
#define UDF_SHED_SEQWRITING     4
#define UDF_SHED_IDLE           5                       /* refcnt'd */
#define UDF_SHED_FREE           6                       /* recycleable */
#define UDF_SHED_MAX            6+1

/* flags */
#define ECC_LOCKED              0x01                    /* prevent access   */
#define ECC_WANTED              0x02                    /* trying access    */
#define ECC_SEQWRITING          0x04                    /* sequential queue */
#define ECC_FLOATING            0x08                    /* not queued yet   */

#define ECC_WAITTIME            10


TAILQ_HEAD(ecclineq, udf_eccline);
struct udf_eccline {
        struct udf_mount         *ump;
        uint64_t                  present;              /* preserve these */
        uint64_t                  readin;               /* bitmap */
        uint64_t                  dirty;                /* bitmap */
        uint64_t                  error;                /* bitmap */
        uint32_t                  refcnt;

        struct timespec           wait_time;
        uint32_t                  flags;
        uint32_t                  start_sector;         /* physical */

        const char               *fname;
        int                       sline;

        struct buf               *buf;
        void                     *blob;

        struct buf               *bufs[UDF_MAX_PACKET_SIZE];
        uint32_t                  bufs_bpos[UDF_MAX_PACKET_SIZE];
        int                       bufs_len[UDF_MAX_PACKET_SIZE];

        int                       queued_on;            /* on which BUFQ list */
        LIST_ENTRY(udf_eccline)   hashchain;            /* on sector lookup  */
};


struct strat_private {
        lwp_t                    *queue_lwp;
        kcondvar_t                discstrat_cv;         /* to wait on       */
        kmutex_t                  discstrat_mutex;      /* disc strategy    */
        kmutex_t                  seqwrite_mutex;       /* protect mappings */

        int                       thread_running;       /* thread control */
        int                       run_thread;           /* thread control */
        int                       thread_finished;      /* thread control */
        int                       cur_queue;

        int                       num_floating;
        int                       num_queued[UDF_SHED_MAX];
        struct bufq_state        *queues[UDF_SHED_MAX];
        struct timespec           last_queued[UDF_SHED_MAX];
        struct disk_strategy      old_strategy_setting;

        struct pool               eccline_pool;
        struct pool               ecclineblob_pool;
        LIST_HEAD(, udf_eccline)  eccline_hash[UDF_ECCBUF_HASHSIZE];
};

/* --------------------------------------------------------------------- */

#define UDF_LOCK_ECCLINE(eccline) udf_lock_eccline(eccline, __FILE__, __LINE__)
#define UDF_UNLOCK_ECCLINE(eccline) udf_unlock_eccline(eccline, __FILE__, __LINE__)

/* can be called with or without discstrat lock */
static void
udf_lock_eccline(struct udf_eccline *eccline, const char *fname, int sline)
{
        struct strat_private *priv = PRIV(eccline->ump);
        int waslocked, ret;

        KASSERT(mutex_owned(&priv->discstrat_mutex));

        waslocked = mutex_owned(&priv->discstrat_mutex);
        if (!waslocked)
                mutex_enter(&priv->discstrat_mutex);

        /* wait until its unlocked first */
        eccline->refcnt++;
        while (eccline->flags & ECC_LOCKED) {
                DPRINTF(ECCLINE, ("waiting for lock at %s:%d\n",
                                        fname, sline));
                DPRINTF(ECCLINE, ("was locked at %s:%d\n",
                                        eccline->fname, eccline->sline));
                eccline->flags |= ECC_WANTED;
                ret = cv_timedwait(&priv->discstrat_cv, &priv->discstrat_mutex,
                        hz/8);
                if (ret == EWOULDBLOCK)
                        DPRINTF(LOCKING, ("eccline lock held, waiting for "
                                "release"));
        }
        eccline->flags |= ECC_LOCKED;
        eccline->flags &= ~ECC_WANTED;
        eccline->refcnt--;

        eccline->fname = fname;
        eccline->sline = sline;

        if (!waslocked)
                mutex_exit(&priv->discstrat_mutex);
}


/* can be called with or without discstrat lock */
static void
udf_unlock_eccline(struct udf_eccline *eccline, const char *fname, int sline)
{
        struct strat_private *priv = PRIV(eccline->ump);
        int waslocked;

        KASSERT(mutex_owned(&priv->discstrat_mutex));

        waslocked = mutex_owned(&priv->discstrat_mutex);
        if (!waslocked)
                mutex_enter(&priv->discstrat_mutex);

        eccline->flags &= ~ECC_LOCKED;
        cv_broadcast(&priv->discstrat_cv);

        if (!waslocked)
                mutex_exit(&priv->discstrat_mutex);
}


/* NOTE discstrat_mutex should be held! */
static void
udf_dispose_eccline(struct udf_eccline *eccline)
{
        struct strat_private *priv = PRIV(eccline->ump);

        KASSERT(mutex_owned(&priv->discstrat_mutex));

        DPRINTF(ECCLINE, ("dispose eccline with start sector %d, "
                "present %0"PRIx64"\n", eccline->start_sector,
                eccline->present));

        KASSERT(eccline->refcnt == 0);
        KASSERT(eccline->dirty  == 0);
        KASSERT(eccline->queued_on == 0);
        KASSERT(eccline->flags & ECC_FLOATING);
        KASSERT(eccline->flags & ECC_LOCKED);

        LIST_REMOVE(eccline, hashchain);
        priv->num_floating--;

        putiobuf(eccline->buf);
        pool_put(&priv->ecclineblob_pool, eccline->blob);
        pool_put(&priv->eccline_pool, eccline);
}


/* NOTE discstrat_mutex should be held! */
static void
udf_push_eccline(struct udf_eccline *eccline, int newqueue)
{
        struct strat_private *priv = PRIV(eccline->ump);

        KASSERT(mutex_owned(&priv->discstrat_mutex));

        DPRINTF(PARANOIA, ("DEBUG: buf %p pushed on queue %d\n", eccline->buf, newqueue));

        KASSERT(eccline->queued_on == 0);
        KASSERT(eccline->flags & ECC_FLOATING);

        /* set buffer block numbers to make sure its queued correctly */
        eccline->buf->b_lblkno   = eccline->start_sector;
        eccline->buf->b_blkno    = eccline->start_sector;
        eccline->buf->b_rawblkno = eccline->start_sector;

        vfs_timestamp(&priv->last_queued[newqueue]);
        eccline->flags &= ~ECC_FLOATING;
        priv->num_floating--;
        eccline->queued_on = newqueue;
        priv->num_queued[newqueue]++;
        bufq_put(priv->queues[newqueue], eccline->buf);

        UDF_UNLOCK_ECCLINE(eccline);

        /* XXX tickle disc strategy statemachine */
        if (newqueue != UDF_SHED_IDLE)
                cv_signal(&priv->discstrat_cv);
}


static struct udf_eccline *
udf_peek_eccline(struct strat_private *priv, int queued_on)
{
        struct udf_eccline *eccline;
        struct buf *buf;

        KASSERT(mutex_owned(&priv->discstrat_mutex));

        for(;;) {
                buf = bufq_peek(priv->queues[queued_on]);
                /* could have been a race, but we'll revisit later */
                if (buf == NULL)
                        return NULL;

                eccline = BTOE(buf);
                UDF_LOCK_ECCLINE(eccline);

                /* might have changed before we obtained the lock */
                if (eccline->queued_on == queued_on)
                        break;

                UDF_UNLOCK_ECCLINE(eccline);
        }

        KASSERT(eccline->queued_on == queued_on);
        KASSERT((eccline->flags & ECC_FLOATING) == 0);

        DPRINTF(PARANOIA, ("DEBUG: buf %p peeked at queue %d\n",
                eccline->buf, queued_on));

        return eccline;
}


static struct udf_eccline *
udf_pop_eccline(struct strat_private *priv, int queued_on)
{
        struct udf_eccline *eccline;
        struct buf *buf;

        KASSERT(mutex_owned(&priv->discstrat_mutex));

        for(;;) {
                buf = bufq_get(priv->queues[queued_on]);
                if (buf == NULL) {
                        // KASSERT(priv->num_queued[queued_on] == 0);
                        return NULL;
                }

                eccline = BTOE(buf);
                UDF_LOCK_ECCLINE(eccline);

                /* might have changed before we obtained the lock */
                if (eccline->queued_on == queued_on)
                        break;

                UDF_UNLOCK_ECCLINE(eccline);
        }

        KASSERT(eccline->queued_on == queued_on);
        KASSERT((eccline->flags & ECC_FLOATING) == 0);

        priv->num_queued[queued_on]--;
        eccline->queued_on = 0;

        eccline->flags |= ECC_FLOATING;
        priv->num_floating++;

        DPRINTF(PARANOIA, ("DEBUG: buf %p popped from queue %d\n",
                eccline->buf, queued_on));

        return eccline;
}


static void
udf_unqueue_eccline(struct strat_private *priv, struct udf_eccline *eccline)
{
        struct buf *ret __diagused;

        UDF_LOCK_ECCLINE(eccline);
        if (eccline->queued_on == 0) {
                KASSERT(eccline->flags & ECC_FLOATING);
                return;
        }

        ret = bufq_cancel(priv->queues[eccline->queued_on], eccline->buf);
        KASSERT(ret == eccline->buf);

        priv->num_queued[eccline->queued_on]--;
        eccline->queued_on = 0;

        eccline->flags |= ECC_FLOATING;
        priv->num_floating++;
}


static struct udf_eccline *
udf_geteccline(struct udf_mount *ump, uint32_t sector, int flags)
{
        struct strat_private *priv = PRIV(ump);
        struct udf_eccline *eccline;
        uint32_t start_sector, lb_size, blobsize;
        uint8_t *eccline_blob;
        int line, line_offset;
        int num_busy;

        mutex_enter(&priv->discstrat_mutex);

        /* lookup in our line cache hashtable */
        line_offset  = sector % ump->packet_size;
        start_sector = sector - line_offset;
        line = (start_sector/ump->packet_size) & UDF_ECCBUF_HASHMASK;

        KASSERT(priv->thread_running);

retry:
        DPRINTF(ECCLINE, ("get line sector %d, line %d\n", sector, line));
        LIST_FOREACH(eccline, &priv->eccline_hash[line], hashchain) {
                if (eccline->start_sector == start_sector) {
                        DPRINTF(ECCLINE, ("\tfound eccline, start_sector %d\n",
                                eccline->start_sector));
                        udf_unqueue_eccline(priv, eccline);

                        mutex_exit(&priv->discstrat_mutex);
                        return eccline;
                }
        }

        /* not found in eccline cache */
        DPRINTF(ECCLINE, ("\tnot found in eccline cache\n"));

        lb_size  = udf_rw32(ump->logical_vol->lb_size);
        blobsize = ump->packet_size * lb_size;

        /* dont allow too many pending requests */
        DPRINTF(ECCLINE, ("\tallocating new eccline\n"));
        num_busy = (priv->num_queued[UDF_SHED_SEQWRITING] + priv->num_floating);
        if ((flags & ECC_SEQWRITING) && (num_busy > UDF_ECCLINE_MAXBUSY)) {
                cv_timedwait(&priv->discstrat_cv,
                        &priv->discstrat_mutex, hz/8);
                goto retry;
        }

        eccline_blob = pool_get(&priv->ecclineblob_pool, PR_NOWAIT);
        eccline = pool_get(&priv->eccline_pool, PR_NOWAIT);
        if ((eccline_blob == NULL) || (eccline == NULL)) {
                if (eccline_blob)
                        pool_put(&priv->ecclineblob_pool, eccline_blob);
                if (eccline)
                        pool_put(&priv->eccline_pool, eccline);

                /* out of memory for now; canibalise freelist */
                eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
                if (eccline == NULL) {
                        /* serious trouble; wait and retry */
                        cv_timedwait(&priv->discstrat_cv,
                                &priv->discstrat_mutex, hz/8);
                        goto retry;
                }

                /* push back line if we're waiting for it or its locked */
                if (eccline->flags & ECC_WANTED) {
                        /* we won a race, but someone else needed it */
                        udf_push_eccline(eccline, UDF_SHED_FREE);
                        goto retry;
                }

                /* unlink this entry */
                LIST_REMOVE(eccline, hashchain);
                KASSERT(eccline->flags & ECC_FLOATING);
                KASSERT(eccline->queued_on == 0);

                eccline_blob = eccline->blob;
                eccline->flags = ECC_FLOATING | ECC_LOCKED;
        } else {
                eccline->flags = ECC_FLOATING | ECC_LOCKED;
                priv->num_floating++;
        }

        eccline->queued_on = 0;
        eccline->blob = eccline_blob;
        eccline->buf  = getiobuf(NULL, true);
        eccline->buf->b_private = eccline;      /* IMPORTANT */

        /* initialise eccline blob */
        /* XXX memset expensive and strictly not needed XXX */
        memset(eccline->blob, 0, blobsize);

        eccline->ump = ump;
        eccline->present = eccline->readin = eccline->dirty = 0;
        eccline->error = 0;
        eccline->refcnt = 0;
        memset(eccline->bufs, 0, UDF_MAX_PACKET_SIZE * sizeof(struct buf *));

        eccline->start_sector    = start_sector;
        eccline->buf->b_lblkno   = start_sector;
        eccline->buf->b_blkno    = start_sector;
        eccline->buf->b_rawblkno = start_sector;

        LIST_INSERT_HEAD(&priv->eccline_hash[line], eccline, hashchain);

        /*
         * TODO possible optimalisation for checking overlap with partitions
         * to get a clue on future eccline usage
         */

        KASSERT(eccline->refcnt == 0);
        KASSERT(eccline->flags & ECC_FLOATING);
        KASSERT(eccline->flags & ECC_LOCKED);
        mutex_exit(&priv->discstrat_mutex);

        return eccline;
}


static void
udf_puteccline(struct udf_eccline *eccline)
{
        struct strat_private *priv = PRIV(eccline->ump);
        struct udf_mount *ump = eccline->ump;
        uint64_t allbits = ((uint64_t) 1 << ump->packet_size)-1;
        int new_queue;

        mutex_enter(&priv->discstrat_mutex);

        DPRINTF(ECCLINE, ("put eccline start sector %d, refcnt %d\n",
                eccline->start_sector, eccline->refcnt));

        KASSERT(eccline->flags & ECC_LOCKED);
        KASSERT(eccline->flags & ECC_FLOATING);

        /* clear all read bits that are already read in */
        if (eccline->readin & eccline->present)
                eccline->readin &= (~eccline->present) & allbits;

        /* if we have active nodes we dont set it on seqwriting */
        if (eccline->refcnt > 1)
                eccline->flags &= ~ECC_SEQWRITING;

        /* select state */
        new_queue = UDF_SHED_FREE;
        if (eccline->refcnt > 0)
                new_queue = UDF_SHED_IDLE;
        if (eccline->flags & ECC_WANTED)
                new_queue = UDF_SHED_IDLE;
        if (eccline->readin)
                new_queue = UDF_SHED_READING;
        if (eccline->dirty) {
                new_queue = UDF_SHED_WAITING;
                vfs_timestamp(&eccline->wait_time);
                eccline->wait_time.tv_sec += ECC_WAITTIME;

                if (eccline->present == allbits) {
                        new_queue = UDF_SHED_WRITING;
                        if (eccline->flags & ECC_SEQWRITING)
                                new_queue = UDF_SHED_SEQWRITING;
                }
        }
        udf_push_eccline(eccline, new_queue);

        mutex_exit(&priv->discstrat_mutex);
}

/* --------------------------------------------------------------------- */

static int
udf_create_nodedscr_rmw(struct udf_strat_args *args)
{
        union dscrptr   **dscrptr  = &args->dscr;
        struct udf_mount *ump      = args->ump;
        struct long_ad   *icb      = args->icb;
        struct udf_eccline *eccline;
        uint64_t bit;
        uint32_t sectornr, lb_size, dummy;
        uint8_t *mem;
        int error, eccsect;

        error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
        if (error)
                return error;

        lb_size  = udf_rw32(ump->logical_vol->lb_size);

        /* get our eccline */
        eccline = udf_geteccline(ump, sectornr, 0);
        eccsect = sectornr - eccline->start_sector;

        bit = (uint64_t) 1 << eccsect;
        eccline->readin  &= ~bit;       /* just in case */
        eccline->present |=  bit;
        eccline->dirty   &= ~bit;       /* Err... euhm... clean? */

        eccline->refcnt++;

        /* clear space */
        mem = ((uint8_t *) eccline->blob) + eccsect * lb_size;
        memset(mem, 0, lb_size);

        udf_puteccline(eccline);

        *dscrptr = (union dscrptr *) mem;
        return 0;
}


static void
udf_free_nodedscr_rmw(struct udf_strat_args *args)
{
        struct udf_mount *ump  = args->ump;
        struct long_ad   *icb  = args->icb;
        struct udf_eccline *eccline;
        uint64_t bit;
        uint32_t sectornr, dummy;
        int error, eccsect;

        error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
        if (error)
                return;

        /* get our eccline */
        eccline = udf_geteccline(ump, sectornr, 0);
        eccsect = sectornr - eccline->start_sector;

        bit = (uint64_t) 1 << eccsect;
        KASSERT(eccline->present & bit);

        eccline->readin &= ~bit;        /* just in case */
        /* XXX eccline->dirty? */

        KASSERT(eccline->refcnt >= 1);
        eccline->refcnt--;

        udf_puteccline(eccline);
}


static int
udf_read_nodedscr_rmw(struct udf_strat_args *args)
{
        union dscrptr   **dscrptr = &args->dscr;
        struct udf_mount *ump = args->ump;
        struct long_ad   *icb = args->icb;
        struct strat_private *priv;
        struct udf_eccline *eccline;
        uint64_t bit;
        uint32_t sectornr, dummy;
        uint8_t *pos;
        int sector_size = ump->discinfo.sector_size;
        int lb_size __diagused = udf_rw32(ump->logical_vol->lb_size);
        int i, error, dscrlen, eccsect;

        KASSERT(sector_size == lb_size);
        error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
        if (error)
                return error;

        /* get our eccline */
        eccline = udf_geteccline(ump, sectornr, 0);
        eccsect = sectornr - eccline->start_sector;

        bit = (uint64_t) 1 << eccsect;
        if ((eccline->present & bit) == 0) {
                /* mark bit for readin */
                eccline->readin |= bit;
                eccline->refcnt++;      /* prevent recycling */
                KASSERT(eccline->bufs[eccsect] == NULL);
                udf_puteccline(eccline);

                /* wait for completion */
                priv = PRIV(eccline->ump);
                mutex_enter(&priv->discstrat_mutex);
                while (((eccline->present | eccline->error) & bit) == 0) {
                        error = cv_timedwait(&priv->discstrat_cv,
                                &priv->discstrat_mutex,
                                hz/8);
                        if (error == EWOULDBLOCK)
                                DPRINTF(LOCKING, ("eccline waiting for read\n"));
                }
                mutex_exit(&priv->discstrat_mutex);

                /* reget our line */
                eccline = udf_geteccline(ump, sectornr, 0);
                KASSERT(eccline->refcnt >= 1);
                eccline->refcnt--;      /* undo refcnt */

                if (eccline->error & bit) {
                        *dscrptr = NULL;
                        udf_puteccline(eccline);
                        return EIO;             /* XXX error code */
                }
        }

        *dscrptr = (union dscrptr *)
                (((uint8_t *) eccline->blob) + eccsect * sector_size);

        /* code from read_phys_descr */
        /* check if its a valid tag */
        error = udf_check_tag(*dscrptr);
        if (error) {
                /* check if its an empty block */
                pos = (uint8_t *) *dscrptr;
                for (i = 0; i < sector_size; i++, pos++) {
                        if (*pos) break;
                }
                if (i == sector_size) {
                        /* return no error but with no dscrptr */
                        error = 0;
                }
                *dscrptr = NULL;
                udf_puteccline(eccline);
                return error;
        }

        /* calculate descriptor size */
        dscrlen = udf_tagsize(*dscrptr, sector_size);
        error = udf_check_tag_payload(*dscrptr, dscrlen);
        if (error) {
                *dscrptr = NULL;
                udf_puteccline(eccline);
                return error;
        }

        /* we have a hold since it has a node descriptor */
        eccline->refcnt++;
        udf_puteccline(eccline);

        return 0;
}


static int
udf_write_nodedscr_rmw(struct udf_strat_args *args)
{
        union dscrptr    *dscrptr = args->dscr;
        struct udf_mount *ump = args->ump;
        struct long_ad   *icb = args->icb;
        struct udf_node *udf_node = args->udf_node;
        struct udf_eccline *eccline;
        uint64_t bit;
        uint32_t sectornr, logsectornr, dummy;
        // int waitfor  = args->waitfor;
        int sector_size = ump->discinfo.sector_size;
        int lb_size __diagused = udf_rw32(ump->logical_vol->lb_size);
        int error, eccsect;

        KASSERT(sector_size == lb_size);
        sectornr    = 0;
        error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
        if (error)
                return error;

        /* get our eccline */
        eccline = udf_geteccline(ump, sectornr, 0);
        eccsect = sectornr - eccline->start_sector;

        bit = (uint64_t) 1 << eccsect;

        /* old callback still pending? */
        if (eccline->bufs[eccsect]) {
                DPRINTF(WRITE, ("udf_write_nodedscr_rmw: writing descriptor"
                                        " over buffer?\n"));
                nestiobuf_done(eccline->bufs[eccsect],
                                eccline->bufs_len[eccsect],
                                0);
                eccline->bufs[eccsect] = NULL;
        }

        /* set sector number in the descriptor and validate */
        dscrptr = (union dscrptr *)
                (((uint8_t *) eccline->blob) + eccsect * sector_size);
        KASSERT(dscrptr == args->dscr);

        logsectornr = udf_rw32(icb->loc.lb_num);
        dscrptr->tag.tag_loc = udf_rw32(logsectornr);
        udf_validate_tag_and_crc_sums(dscrptr);

        udf_fixup_node_internals(ump, (uint8_t *) dscrptr, UDF_C_NODE);

        /* set our flags */
        KASSERT(eccline->present & bit);
        eccline->dirty |= bit;

        KASSERT(udf_tagsize(dscrptr, sector_size) <= sector_size);

        udf_node->outstanding_nodedscr--;
        if (udf_node->outstanding_nodedscr == 0) {
                /* XXX still using wakeup! */
                UDF_UNLOCK_NODE(udf_node, 0);
                cv_broadcast(&udf_node->node_lock);
        }
        udf_puteccline(eccline);

        /* XXX waitfor not used */
        return 0;
}


static void
udf_queuebuf_rmw(struct udf_strat_args *args)
{
        struct udf_mount *ump = args->ump;
        struct buf *buf = args->nestbuf;
        struct desc_tag *tag;
        struct strat_private *priv = PRIV(ump);
        struct udf_eccline *eccline;
        struct long_ad *node_ad_cpy;
        uint64_t bit, *lmapping, *pmapping, *lmappos, *pmappos, blknr;
        uint32_t buf_len, len, sectors, sectornr, our_sectornr;
        uint32_t bpos;
        uint16_t vpart_num;
        uint8_t *fidblk, *src, *dst;
        int sector_size = ump->discinfo.sector_size;
        int blks = sector_size / DEV_BSIZE;
        int eccsect, what, queue, error;

        KASSERT(ump);
        KASSERT(buf);
        KASSERT(buf->b_iodone == nestiobuf_iodone);

        blknr        = buf->b_blkno;
        our_sectornr = blknr / blks;

        what = buf->b_udf_c_type;
        queue = UDF_SHED_READING;
        if ((buf->b_flags & B_READ) == 0) {
                /* writing */
                queue = UDF_SHED_SEQWRITING;
                if (what == UDF_C_ABSOLUTE)
                        queue = UDF_SHED_WRITING;
                if (what == UDF_C_DSCR)
                        queue = UDF_SHED_WRITING;
                if (what == UDF_C_NODE)
                        queue = UDF_SHED_WRITING;
        }

        if (queue == UDF_SHED_READING) {
                DPRINTF(SHEDULE, ("\nudf_queuebuf_rmw READ %p : sector %d type %d,"
                        "b_resid %d, b_bcount %d, b_bufsize %d\n",
                        buf, (uint32_t) buf->b_blkno / blks, buf->b_udf_c_type,
                        buf->b_resid, buf->b_bcount, buf->b_bufsize));

                /* mark bits for reading */
                buf_len = buf->b_bcount;
                sectornr = our_sectornr;
                eccline = udf_geteccline(ump, sectornr, 0);
                eccsect = sectornr - eccline->start_sector;
                bpos = 0;
                while (buf_len) {
                        len = MIN(buf_len, sector_size);
                        if ((eccsect < 0) || (eccsect >= ump->packet_size)) {
                                udf_puteccline(eccline);
                                eccline = udf_geteccline(ump, sectornr, 0);
                                eccsect = sectornr - eccline->start_sector;
                        }
                        bit = (uint64_t) 1 << eccsect;
                        error = eccline->error & bit ? EIO : 0;
                        if (eccline->present & bit) {
                                src = (uint8_t *) eccline->blob +
                                        eccsect * sector_size;
                                dst = (uint8_t *) buf->b_data + bpos;
                                if (!error)
                                        memcpy(dst, src, len);
                                nestiobuf_done(buf, len, error);
                        } else {
                                eccline->readin |= bit;
                                KASSERT(eccline->bufs[eccsect] == NULL);
                                eccline->bufs[eccsect] = buf;
                                eccline->bufs_bpos[eccsect] = bpos;
                                eccline->bufs_len[eccsect] = len;
                        }
                        bpos += sector_size;
                        eccsect++;
                        sectornr++;
                        buf_len -= len;
                }
                udf_puteccline(eccline);
                return;
        }

        if (queue == UDF_SHED_WRITING) {
                DPRINTF(SHEDULE, ("\nudf_queuebuf_rmw WRITE %p : sector %d "
                        "type %d, b_resid %d, b_bcount %d, b_bufsize %d\n",
                        buf, (uint32_t) buf->b_blkno / blks, buf->b_udf_c_type,
                        buf->b_resid, buf->b_bcount, buf->b_bufsize));

                /* if we have FIDs fixup using buffer's sector number(s) */
                if (buf->b_udf_c_type == UDF_C_FIDS)
                        panic("UDF_C_FIDS in SHED_WRITING!\n");

                udf_fixup_node_internals(ump, buf->b_data, buf->b_udf_c_type);

                /* copy parts into the bufs and set for writing */
                buf_len = buf->b_bcount;
                sectornr = our_sectornr;
                eccline = udf_geteccline(ump, sectornr, 0);
                eccsect = sectornr - eccline->start_sector;
                bpos = 0;
                while (buf_len) {
                        len = MIN(buf_len, sector_size);
                        if ((eccsect < 0) || (eccsect >= ump->packet_size)) {
                                udf_puteccline(eccline);
                                eccline = udf_geteccline(ump, sectornr, 0);
                                eccsect = sectornr - eccline->start_sector;
                        }
                        bit = (uint64_t) 1 << eccsect;
                        KASSERT((eccline->readin & bit) == 0);
                        eccline->present |= bit;
                        eccline->dirty   |= bit;
                        if (eccline->bufs[eccsect]) {
                                /* old callback still pending */
                                nestiobuf_done(eccline->bufs[eccsect],
                                                eccline->bufs_len[eccsect],
                                                0);
                                eccline->bufs[eccsect] = NULL;
                        }

                        src = (uint8_t *) buf->b_data + bpos;
                        dst = (uint8_t *) eccline->blob + eccsect * sector_size;
                        if (len != sector_size)
                                memset(dst, 0, sector_size);
                        memcpy(dst, src, len);

                        /* note that its finished for this extent */
                        eccline->bufs[eccsect] = NULL;
                        nestiobuf_done(buf, len, 0);

                        bpos += sector_size;
                        eccsect++;
                        sectornr++;
                        buf_len -= len;
                }
                udf_puteccline(eccline);
                return;

        }

        /* sequential writing */
        KASSERT(queue == UDF_SHED_SEQWRITING);
        DPRINTF(SHEDULE, ("\nudf_queuebuf_rmw SEQWRITE %p : sector XXXX "
                "type %d, b_resid %d, b_bcount %d, b_bufsize %d\n",
                buf, buf->b_udf_c_type, buf->b_resid, buf->b_bcount,
                buf->b_bufsize));
        /*
         * Buffers should not have been allocated to disc addresses yet on
         * this queue. Note that a buffer can get multiple extents allocated.
         * Note that it *looks* like the normal writing but its different in
         * the details.
         *
         * lmapping contains lb_num relative to base partition.
         *
         * XXX should we try to claim/organize the allocated memory to
         * block-aligned pieces?
         */
        mutex_enter(&priv->seqwrite_mutex);

        lmapping    = ump->la_lmapping;
        node_ad_cpy = ump->la_node_ad_cpy;

        /* logically allocate buf and map it in the file */
        udf_late_allocate_buf(ump, buf, lmapping, node_ad_cpy, &vpart_num);

        /* if we have FIDs, fixup using the new allocation table */
        if (buf->b_udf_c_type == UDF_C_FIDS) {
                buf_len = buf->b_bcount;
                bpos = 0;
                lmappos = lmapping;
                while (buf_len) {
                        sectornr = *lmappos++;
                        len = MIN(buf_len, sector_size);
                        fidblk = (uint8_t *) buf->b_data + bpos;
                        udf_fixup_fid_block(fidblk, sector_size,
                                0, len, sectornr);
                        bpos += len;
                        buf_len -= len;
                }
        }
        if (buf->b_udf_c_type == UDF_C_METADATA_SBM) {
                if (buf->b_lblkno == 0) {
                        /* update the tag location inside */
                        tag = (struct desc_tag *) buf->b_data;
                        tag->tag_loc = udf_rw32(*lmapping);
                        udf_validate_tag_and_crc_sums(buf->b_data);
                }
        }
        udf_fixup_node_internals(ump, buf->b_data, buf->b_udf_c_type);

        /*
         * Translate new mappings in lmapping to pmappings.
         * pmapping to contain lb_nums as used for disc addressing.
         */
        pmapping = ump->la_pmapping;
        sectors  = (buf->b_bcount + sector_size -1) / sector_size;
        udf_translate_vtop_list(ump, sectors, vpart_num, lmapping, pmapping);

        /* copy parts into the bufs and set for writing */
        pmappos = pmapping;
        buf_len = buf->b_bcount;
        sectornr = *pmappos++;
        eccline = udf_geteccline(ump, sectornr, ECC_SEQWRITING);
        eccsect = sectornr - eccline->start_sector;
        bpos = 0;
        while (buf_len) {
                len = MIN(buf_len, sector_size);
                eccsect = sectornr - eccline->start_sector;
                if ((eccsect < 0) || (eccsect >= ump->packet_size)) {
                        eccline->flags |= ECC_SEQWRITING;
                        udf_puteccline(eccline);
                        eccline = udf_geteccline(ump, sectornr, ECC_SEQWRITING);
                        eccsect = sectornr - eccline->start_sector;
                }
                bit = (uint64_t) 1 << eccsect;
                KASSERT((eccline->readin & bit) == 0);
                eccline->present |= bit;
                eccline->dirty   |= bit;
                eccline->bufs[eccsect] = NULL;

                src = (uint8_t *) buf->b_data + bpos;
                dst = (uint8_t *)
                        eccline->blob + eccsect * sector_size;
                if (len != sector_size)
                        memset(dst, 0, sector_size);
                memcpy(dst, src, len);

                /* note that its finished for this extent */
                nestiobuf_done(buf, len, 0);

                bpos += sector_size;
                sectornr = *pmappos++;
                buf_len -= len;
        }
        eccline->flags |= ECC_SEQWRITING;
        udf_puteccline(eccline);
        mutex_exit(&priv->seqwrite_mutex);
}

/* --------------------------------------------------------------------- */

static void
udf_sync_caches_rmw(struct udf_strat_args *args)
{
        struct udf_mount *ump = args->ump;

        udf_mmc_synchronise_caches(ump);
}

/* --------------------------------------------------------------------- */

static void
udf_shedule_read_callback(struct buf *buf)
{
        struct udf_eccline *eccline = BTOE(buf);
        struct udf_mount *ump = eccline->ump;
        uint64_t bit;
        uint8_t *src, *dst;
        int sector_size = ump->discinfo.sector_size;
        int error, i, len;

        DPRINTF(ECCLINE, ("read callback called on buf %p\n", buf));

        /* post process read action */
        KASSERT(eccline->flags & ECC_LOCKED);
        error = buf->b_error;
        for (i = 0; i < ump->packet_size; i++) {
                bit = (uint64_t) 1 << i;
                src = (uint8_t *) buf->b_data +   i * sector_size;
                dst = (uint8_t *) eccline->blob + i * sector_size;
                if (eccline->present & bit)
                        continue;
                eccline->present |= bit;
                if (error)
                        eccline->error |= bit;
                if (eccline->bufs[i]) {
                        dst = (uint8_t *) eccline->bufs[i]->b_data +
                                eccline->bufs_bpos[i];
                        len = eccline->bufs_len[i];
                        if (!error)
                                memcpy(dst, src, len);
                        nestiobuf_done(eccline->bufs[i], len, error);
                        eccline->bufs[i] = NULL;
                }

        }
        KASSERT(buf->b_data == eccline->blob);
        KASSERT(eccline->present == ((uint64_t) 1 << ump->packet_size)-1);

        /*
         * XXX TODO what to do on read errors? read in all sectors
         * synchronously and allocate a sparable entry?
         */

        udf_puteccline(eccline);
        DPRINTF(ECCLINE, ("read callback finished\n"));
}


static void
udf_shedule_write_callback(struct buf *buf)
{
        struct udf_eccline *eccline = BTOE(buf);
        struct udf_mount *ump = eccline->ump;
        uint64_t bit;
        int error, i;

        DPRINTF(ECCLINE, ("write callback called on buf %p\n", buf));

        /* post process write action */
        KASSERT(eccline->flags & ECC_LOCKED);
        error = buf->b_error;
        for (i = 0; i < ump->packet_size; i++) {
                bit = (uint64_t) 1 << i;
                if ((eccline->dirty & bit) == 0)
                        continue;
                if (error) {
                        eccline->error |= bit;
                } else {
                        eccline->dirty &= ~bit;
                }

                KASSERT(eccline->bufs[i] == 0);
        }
        KASSERT(eccline->dirty == 0);
        KASSERT(error == 0);

        /*
         * XXX TODO on write errors allocate a sparable entry and reissue
         */

        udf_puteccline(eccline);
        DPRINTF(ECCLINE, ("write callback finished\n"));
}


static void
udf_issue_eccline(struct udf_eccline *eccline, int queued_on)
{
        struct udf_mount *ump = eccline->ump;
        struct strat_private *priv = PRIV(ump);
        struct buf *buf, *nestbuf;
        uint64_t bit, allbits = ((uint64_t) 1 << ump->packet_size)-1;
        uint32_t start;
        int sector_size = ump->discinfo.sector_size;
        int blks = sector_size / DEV_BSIZE;
        int i;

        KASSERT(eccline->flags & ECC_LOCKED);

        if (queued_on == UDF_SHED_READING) {
                DPRINTF(SHEDULE, ("udf_issue_eccline reading : "));
                /* read all bits that are not yet present */
                eccline->readin = (~eccline->present) & allbits;
                KASSERT(eccline->readin);
                start = eccline->start_sector;
                buf = eccline->buf;
                buf->b_flags    = B_READ | B_ASYNC;
                SET(buf->b_cflags, BC_BUSY);    /* mark buffer busy */
                buf->b_oflags   = 0;
                buf->b_iodone   = udf_shedule_read_callback;
                buf->b_data     = eccline->blob;
                buf->b_bcount   = ump->packet_size * sector_size;
                buf->b_resid    = buf->b_bcount;
                buf->b_bufsize  = buf->b_bcount;
                buf->b_private  = eccline;
                BIO_SETPRIO(buf, BPRIO_DEFAULT);
                buf->b_lblkno   = buf->b_blkno = buf->b_rawblkno = start * blks;
                buf->b_proc     = NULL;

                if (eccline->present != 0) {
                        for (i = 0; i < ump->packet_size; i++) {
                                bit = (uint64_t) 1 << i;
                                if (eccline->present & bit) {
                                        nestiobuf_done(buf, sector_size, 0);
                                        continue;
                                }
                                nestbuf = getiobuf(NULL, true);
                                nestiobuf_setup(buf, nestbuf, i * sector_size,
                                        sector_size);
                                /* adjust blocknumber to read */
                                nestbuf->b_blkno = buf->b_blkno + i*blks;
                                nestbuf->b_rawblkno = buf->b_rawblkno + i*blks;

                                DPRINTF(SHEDULE, ("sector %d ", start + i));

                                /* mutex dance since it could lock */
                                mutex_exit(&priv->discstrat_mutex);
                                        /* call asynchronous */
                                        VOP_STRATEGY(ump->devvp, nestbuf);
                                mutex_enter(&priv->discstrat_mutex);
                        }
                        DPRINTF(SHEDULE, ("\n"));
                        return;
                }
        } else {
                /* write or seqwrite */
                DPRINTF(SHEDULE, ("udf_issue_eccline writing or seqwriting : "));
                DPRINTF(SHEDULE, ("\n\tpresent %"PRIx64", readin %"PRIx64", "
                        "dirty %"PRIx64"\n\t", eccline->present, eccline->readin,
                        eccline->dirty));
                KASSERT(eccline->present == allbits);

                start = eccline->start_sector;
                buf = eccline->buf;
                buf->b_flags    = B_WRITE | B_ASYNC;
                SET(buf->b_cflags, BC_BUSY);    /* mark buffer busy */
                buf->b_oflags   = 0;
                buf->b_iodone   = udf_shedule_write_callback;
                buf->b_data     = eccline->blob;
                buf->b_bcount   = ump->packet_size * sector_size;
                buf->b_resid    = buf->b_bcount;
                buf->b_bufsize  = buf->b_bcount;
                buf->b_private  = eccline;
                BIO_SETPRIO(buf, BPRIO_DEFAULT);
                buf->b_lblkno   = buf->b_blkno = buf->b_rawblkno = start * blks;
                buf->b_proc     = NULL;
        }

        /* mutex dance since it could lock */
        mutex_exit(&priv->discstrat_mutex);
                /* call asynchronous */
                DPRINTF(SHEDULE, ("sector %d for %d\n",
                        start, ump->packet_size));
                VOP_STRATEGY(ump->devvp, buf);
        mutex_enter(&priv->discstrat_mutex);
}


static void
udf_discstrat_thread(void *arg)
{
        struct udf_mount *ump = (struct udf_mount *) arg;
        struct strat_private *priv = PRIV(ump);
        struct udf_eccline *eccline;
        struct timespec now, *last;
        uint64_t allbits = ((uint64_t) 1 << ump->packet_size)-1;
        int new_queue, wait, work;

        work = 1;
        priv->thread_running = 1;
        cv_broadcast(&priv->discstrat_cv);

        mutex_enter(&priv->discstrat_mutex);
        priv->num_floating = 0;
        while (priv->run_thread || work || priv->num_floating) {
                /* get our time */
                vfs_timestamp(&now);

                /* maintenance: handle eccline state machine */
                for(;;) {
                        /* only peek at it */
                        eccline = udf_peek_eccline(priv, UDF_SHED_WAITING);
                        if (eccline == NULL)
                                break;

                        /* if not reading, wait until the time has come */
                        if ((priv->cur_queue != UDF_SHED_READING) &&
                                (eccline->wait_time.tv_sec - now.tv_sec > 0)) {
                                        UDF_UNLOCK_ECCLINE(eccline);
                                        /* all others are later, so break off */
                                        break;
                        }

                        /* release */
                        UDF_UNLOCK_ECCLINE(eccline);

                        /* do get it */
                        eccline = udf_pop_eccline(priv, UDF_SHED_WAITING);

                        /* requeue according to state */
                        new_queue = UDF_SHED_FREE;      /* unlikely */
                        if (eccline->refcnt > 0)
                                new_queue = UDF_SHED_IDLE;
                        if (eccline->flags & ECC_WANTED)
                                new_queue = UDF_SHED_IDLE;
                        if (eccline->readin)
                                new_queue = UDF_SHED_READING;
                        if (eccline->dirty) {
                                new_queue = UDF_SHED_READING;
                                if (eccline->present == allbits) {
                                        new_queue = UDF_SHED_WRITING;
                                        if (eccline->flags & ECC_SEQWRITING)
                                                new_queue = UDF_SHED_SEQWRITING;
                                }
                        }
                        udf_push_eccline(eccline, new_queue);
                }

                /* maintenance: free excess ecclines */
                while (priv->num_queued[UDF_SHED_FREE] > UDF_ECCLINE_MAXFREE) {
                        eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
                        KASSERT(eccline);
                        KASSERT(eccline->refcnt == 0);
                        if (eccline->flags & ECC_WANTED) {
                                /* we won the race, but we dont want to win */
                                DPRINTF(ECCLINE, ("Tried removing, pushed back to free list\n"));
                                udf_push_eccline(eccline, UDF_SHED_IDLE);
                        } else {
                                DPRINTF(ECCLINE, ("Removing entry from free list\n"));
                                udf_dispose_eccline(eccline);
                        }
                }

                /* process the current selected queue */
                /* get our time */
                vfs_timestamp(&now);
                last = &priv->last_queued[priv->cur_queue];

                /* get our line */
                eccline = udf_pop_eccline(priv, priv->cur_queue);
                if (eccline) {
                        wait = 0;
                        new_queue = priv->cur_queue;
                        DPRINTF(ECCLINE, ("UDF_ISSUE_ECCLINE\n"));

                        udf_issue_eccline(eccline, priv->cur_queue);
                } else {
                        /* don't switch too quickly */
                        if (now.tv_sec - last->tv_sec < 2) {
                                /* wait some time */
                                cv_timedwait(&priv->discstrat_cv,
                                        &priv->discstrat_mutex, hz);
                                /* we assume there is work to be done */
                                work = 1;
                                continue;
                        }

                        /* XXX select on queue lengths ? */
                        wait = 1;
                        /* check if we can/should switch */
                        new_queue = priv->cur_queue;
                        if (bufq_peek(priv->queues[UDF_SHED_READING]))
                                new_queue = UDF_SHED_READING;
                        if (bufq_peek(priv->queues[UDF_SHED_WRITING]))
                                new_queue = UDF_SHED_WRITING;
                        if (bufq_peek(priv->queues[UDF_SHED_SEQWRITING]))
                                new_queue = UDF_SHED_SEQWRITING;
                }

                /* give room */
                mutex_exit(&priv->discstrat_mutex);

                if (new_queue != priv->cur_queue) {
                        wait = 0;
                        DPRINTF(SHEDULE, ("switching from %d to %d\n",
                                priv->cur_queue, new_queue));
                        priv->cur_queue = new_queue;
                }
                mutex_enter(&priv->discstrat_mutex);

                /* wait for more if needed */
                if (wait)
                        cv_timedwait(&priv->discstrat_cv,
                                &priv->discstrat_mutex, hz/4);  /* /8 */

                work  = (bufq_peek(priv->queues[UDF_SHED_WAITING]) != NULL);
                work |= (bufq_peek(priv->queues[UDF_SHED_READING]) != NULL);
                work |= (bufq_peek(priv->queues[UDF_SHED_WRITING]) != NULL);
                work |= (bufq_peek(priv->queues[UDF_SHED_SEQWRITING]) != NULL);

                DPRINTF(PARANOIA, ("work : (%d, %d, %d) -> work %d, float %d\n",
                        (bufq_peek(priv->queues[UDF_SHED_READING]) != NULL),
                        (bufq_peek(priv->queues[UDF_SHED_WRITING]) != NULL),
                        (bufq_peek(priv->queues[UDF_SHED_SEQWRITING]) != NULL),
                        work, priv->num_floating));
        }

        mutex_exit(&priv->discstrat_mutex);

        /* tear down remaining ecclines */
        mutex_enter(&priv->discstrat_mutex);
        KASSERT(bufq_peek(priv->queues[UDF_SHED_WAITING]) == NULL);
        KASSERT(bufq_peek(priv->queues[UDF_SHED_IDLE]) == NULL);
        KASSERT(bufq_peek(priv->queues[UDF_SHED_READING]) == NULL);
        KASSERT(bufq_peek(priv->queues[UDF_SHED_WRITING]) == NULL);
        KASSERT(bufq_peek(priv->queues[UDF_SHED_SEQWRITING]) == NULL);

        KASSERT(priv->num_queued[UDF_SHED_WAITING] == 0);
        KASSERT(priv->num_queued[UDF_SHED_IDLE] == 0);
        KASSERT(priv->num_queued[UDF_SHED_READING] == 0);
        KASSERT(priv->num_queued[UDF_SHED_WRITING] == 0);
        KASSERT(priv->num_queued[UDF_SHED_SEQWRITING] == 0);

        eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
        while (eccline) {
                udf_dispose_eccline(eccline);
                eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
        }
        KASSERT(priv->num_queued[UDF_SHED_FREE] == 0);
        mutex_exit(&priv->discstrat_mutex);

        priv->thread_running  = 0;
        priv->thread_finished = 1;
        cv_broadcast(&priv->discstrat_cv);

        kthread_exit(0);
        /* not reached */
}

/* --------------------------------------------------------------------- */

/*
 * Buffer memory pool allocator.
 */

static void *
ecclinepool_page_alloc(struct pool *pp, int flags)
{
        return (void *)uvm_km_alloc(kernel_map,
            MAXBSIZE, MAXBSIZE,
            ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK)
                | UVM_KMF_WIRED /* UVM_KMF_PAGABLE? */);
}

static void
ecclinepool_page_free(struct pool *pp, void *v)
{
        uvm_km_free(kernel_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED);
}

static struct pool_allocator ecclinepool_allocator = {
        .pa_alloc = ecclinepool_page_alloc,
        .pa_free  = ecclinepool_page_free,
        .pa_pagesz = MAXBSIZE,
};


static void
udf_discstrat_init_rmw(struct udf_strat_args *args)
{
        struct udf_mount *ump = args->ump;
        struct strat_private *priv = PRIV(ump);
        uint32_t lb_size, blobsize, hashline;
        int i;

        KASSERT(ump);
        KASSERT(ump->logical_vol);
        KASSERT(priv == NULL);

        lb_size = udf_rw32(ump->logical_vol->lb_size);
        blobsize = ump->packet_size * lb_size;
        KASSERT(lb_size > 0);
        KASSERT(ump->packet_size <= 64);

        /* initialise our memory space */
        ump->strategy_private = malloc(sizeof(struct strat_private),
                M_UDFTEMP, M_WAITOK);
        priv = ump->strategy_private;
        memset(priv, 0 , sizeof(struct strat_private));

        /* initialise locks */
        cv_init(&priv->discstrat_cv, "udfstrat");
        mutex_init(&priv->discstrat_mutex, MUTEX_DEFAULT, IPL_NONE);
        mutex_init(&priv->seqwrite_mutex, MUTEX_DEFAULT, IPL_NONE);

        /* initialise struct eccline pool */
        pool_init(&priv->eccline_pool, sizeof(struct udf_eccline),
                0, 0, 0, "udf_eccline_pool", NULL, IPL_NONE);

        /* initialise eccline blob pool */
        ecclinepool_allocator.pa_pagesz = blobsize;
        pool_init(&priv->ecclineblob_pool, blobsize,
                0, 0, 0, "udf_eccline_blob", &ecclinepool_allocator, IPL_NONE);

        /* initialise main queues */
        for (i = 0; i < UDF_SHED_MAX; i++) {
                priv->num_queued[i] = 0;
                vfs_timestamp(&priv->last_queued[i]);
        }
        bufq_alloc(&priv->queues[UDF_SHED_WAITING], "fcfs",
                BUFQ_SORT_RAWBLOCK);
        bufq_alloc(&priv->queues[UDF_SHED_READING], "disksort",
                BUFQ_SORT_RAWBLOCK);
        bufq_alloc(&priv->queues[UDF_SHED_WRITING], "disksort",
                BUFQ_SORT_RAWBLOCK);
        bufq_alloc(&priv->queues[UDF_SHED_SEQWRITING], "disksort", 0);

        /* initialise administrative queues */
        bufq_alloc(&priv->queues[UDF_SHED_IDLE], "fcfs", 0);
        bufq_alloc(&priv->queues[UDF_SHED_FREE], "fcfs", 0);

        for (hashline = 0; hashline < UDF_ECCBUF_HASHSIZE; hashline++) {
                LIST_INIT(&priv->eccline_hash[hashline]);
        }

        /* create our disk strategy thread */
        priv->cur_queue = UDF_SHED_READING;
        priv->thread_finished = 0;
        priv->thread_running  = 0;
        priv->run_thread      = 1;
        if (kthread_create(PRI_NONE, 0 /* KTHREAD_MPSAFE*/, NULL /* cpu_info*/,
                udf_discstrat_thread, ump, &priv->queue_lwp,
                "%s", "udf_rw")) {
                panic("fork udf_rw");
        }

        /* wait for thread to spin up */
        mutex_enter(&priv->discstrat_mutex);
        while (!priv->thread_running) {
                cv_timedwait(&priv->discstrat_cv, &priv->discstrat_mutex, hz);
        }
        mutex_exit(&priv->discstrat_mutex);
}


static void
udf_discstrat_finish_rmw(struct udf_strat_args *args)
{
        struct udf_mount *ump = args->ump;
        if (ump == NULL)
                return;

        struct strat_private *priv = PRIV(ump);

        /* stop our sheduling thread */
        KASSERT(priv->run_thread == 1);
        priv->run_thread = 0;

        mutex_enter(&priv->discstrat_mutex);
        while (!priv->thread_finished) {
                cv_broadcast(&priv->discstrat_cv);
                cv_timedwait(&priv->discstrat_cv, &priv->discstrat_mutex, hz);
        }
        mutex_exit(&priv->discstrat_mutex);

        /* kthread should be finished now */
        cv_destroy(&priv->discstrat_cv);
        mutex_destroy(&priv->discstrat_mutex);
        mutex_destroy(&priv->seqwrite_mutex);

        /* cleanup our pools */
        pool_destroy(&priv->eccline_pool);
        pool_destroy(&priv->ecclineblob_pool);

        /* free our private space */
        free(ump->strategy_private, M_UDFTEMP);
        ump->strategy_private = NULL;
}

/* --------------------------------------------------------------------- */

struct udf_strategy udf_strat_rmw =
{
        udf_create_nodedscr_rmw,
        udf_free_nodedscr_rmw,
        udf_read_nodedscr_rmw,
        udf_write_nodedscr_rmw,
        udf_queuebuf_rmw,
        udf_sync_caches_rmw,
        udf_discstrat_init_rmw,
        udf_discstrat_finish_rmw
};