/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/mode.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/varargs.h>
#include <sys/time.h>
#include <sys/buf.h>
#include <sys/kmem.h>
#include <sys/t_lock.h>
#include <sys/poll.h>
#include <sys/debug.h>
#include <sys/cred.h>
#include <sys/lockfs.h>
#include <sys/fs/ufs_fs.h>
#include <sys/fs/ufs_inode.h>
#include <sys/fs/ufs_panic.h>
#include <sys/fs/ufs_lockfs.h>
#include <sys/fs/ufs_trans.h>
#include <sys/fs/ufs_mount.h>
#include <sys/fs/ufs_prot.h>
#include <sys/fs/ufs_bio.h>
#include <sys/pathname.h>
#include <sys/utsname.h>
#include <sys/conf.h>

/* handy */
#define abs(x)          ((x) < 0? -(x): (x))

#if defined(DEBUG)

#define DBGLVL_NONE     0x00000000
#define DBGLVL_MAJOR    0x00000100
#define DBGLVL_MINOR    0x00000200
#define DBGLVL_MINUTE   0x00000400
#define DBGLVL_TRIVIA   0x00000800
#define DBGLVL_HIDEOUS  0x00001000

#define DBGFLG_NONE             0x00000000
#define DBGFLG_NOPANIC          0x00000001
#define DBGFLG_LVLONLY          0x00000002
#define DBGFLG_FIXWOULDPANIC    0x00000004

#define DBGFLG_FLAGMASK         0x0000000F
#define DBGFLG_LEVELMASK        ~DBGFLG_FLAGMASK

#define DEBUG_FLAGS     (ufs_fix_failure_dbg & DBGFLG_FLAGMASK)
#define DEBUG_LEVEL     (ufs_fix_failure_dbg & DBGFLG_LEVELMASK)

unsigned int ufs_fix_failure_dbg =      DBGLVL_NONE | DBGFLG_NONE;

#define DCALL(dbg_level, call)                                          \
        {                                                               \
                if (DEBUG_LEVEL != DBGLVL_NONE) {                       \
                        if (DEBUG_FLAGS & DBGFLG_LVLONLY) {             \
                                if (DEBUG_LEVEL & dbg_level) {          \
                                        call;                           \
                                }                                       \
                        } else {                                        \
                                if (dbg_level <= DEBUG_LEVEL) {         \
                                        call;                           \
                                }                                       \
                        }                                               \
                }                                                       \
        }

#define DPRINTF(dbg_level, msg)         DCALL(dbg_level, printf msg)

#define MAJOR(msg)                      DPRINTF(DBGLVL_MAJOR, msg)
#define MINOR(msg)                      DPRINTF(DBGLVL_MINOR, msg)
#define MINUTE(msg)                     DPRINTF(DBGLVL_MINUTE, msg)
#define TRIVIA(msg)                     DPRINTF(DBGLVL_TRIVIA, msg)
#define HIDEOUS(msg)                    DPRINTF(DBGLVL_HIDEOUS, msg)

#else   /* !DEBUG */

#define DCALL(ignored_dbg_level, ignored_routine)
#define MAJOR(ignored)
#define MINOR(ignored)
#define MINUTE(ignored)
#define TRIVIA(ignored)
#define HIDEOUS(ignored)

#endif /* DEBUG */

#define NULLSTR(str)    (!(str) || *(str) == '\0'? "<null>" : (str))
#define NULSTRING       ""

/* somewhat arbitrary limits, in seconds */
/* all probably ought to be different, but these are convenient for debugging */
const time_t    UF_TOO_LONG             = 128;  /* max. wait for fsck start */

/* all of these are in units of seconds used for retry period while ... */
const time_t    UF_FIXSTART_PERIOD      = 16;   /* awaiting fsck start */
const time_t    UF_FIXPOLL_PERIOD       = 256;  /* awaiting fsck finish */
const time_t    UF_SHORT_ERROR_PERIOD   = 4;    /* after (lockfs) error */
const time_t    UF_LONG_ERROR_PERIOD    = 512;  /* after (lockfs) error */

#define NO_ERROR                0
#define LOCKFS_OLOCK            LOCKFS_MAXLOCK+1

const ulong_t   GB                      = 1024 * 1024 * 1024;
const ulong_t   SecondsPerGig           = 1024; /* ~17 minutes (overestimate) */

/*
 * per filesystem flags
 */
const int       UFSFX_PANIC             = (UFSMNT_ONERROR_PANIC >> 4);
const int       UFSFX_LCKONLY           = (UFSMNT_ONERROR_LOCK >> 4);
const int       UFSFX_LCKUMOUNT         = (UFSMNT_ONERROR_UMOUNT >> 4);
const int       UFSFX_DEFAULT           = (UFSMNT_ONERROR_DEFAULT >> 4);
const int       UFSFX_REPAIR_START      = 0x10000000;

/* return protocols */

typedef enum triage_return_code {
        TRIAGE_DEAD = -1,
        TRIAGE_NO_SPIRIT,
        TRIAGE_ATTEND_TO
} triage_t;

typedef enum statefunc_return_code {
        SFRC_SUCCESS = 1,
        SFRC_FAIL = 0
} sfrc_t;

/* external references */
/* in ufs_thread.c */
extern int      ufs_thread_run(struct ufs_q *, callb_cpr_t *cprinfop);
extern int      ufs_checkaccton(vnode_t *);             /* in ufs_lockfs.c */
extern int      ufs_checkswapon(vnode_t *);             /* in ufs_lockfs.c */

extern struct pollhead          ufs_pollhd;             /* in ufs_vnops.c */

/* globals */
struct  ufs_q    ufs_fix;

/*
 * patchable constants:
 *   These are set in ufsfx_init() [called at modload]
 */
struct ufs_failure_tunable {
        long     uft_too_long;          /* limit repair startup time */
        long     uft_fixstart_period;   /* pre-repair start period */
        long     uft_fixpoll_period;    /* post-fsck start period */
        long     uft_short_err_period;  /* post-error short period */
        long     uft_long_err_period;   /* post-error long period */
} ufsfx_tune;

/* internal statistics of events */
struct uf_statistics {
        ulong_t         ufst_lock_violations;
        ulong_t         ufst_current_races;
        ulong_t         ufst_unmount_failures;
        ulong_t         ufst_num_fixed;
        ulong_t         ufst_num_failed;
        ulong_t         ufst_cpu_waste;
        time_t          ufst_last_start_tm;
        kmutex_t        ufst_mutex;
} uf_stats;

typedef enum state_action {
        UFA_ERROR = -1,         /* internal error */
        UFA_FOUND,              /* found uf in state */
        UFA_SET                 /* change uf to state */
} ufsa_t;

/* state definition */
typedef struct uf_state_desc {
        int       ud_v;                                 /* value */
        char     *ud_name;                              /* name */
        sfrc_t  (*ud_sfp)(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
                                                        /* per-state actions */
        ufs_failure_states_t      ud_prev;              /* valid prev. states */

        struct uf_state_desc_attr {
                unsigned        terminal:1;     /* no action req. if found */
                unsigned        at_fail:1;      /* state set by thread */
                                                /* encountering the error */
                unsigned        unused;
        } ud_attr;
} ufsd_t;

/*
 * forward references
 */

/* thread to watch for failures */
static void     ufsfx_thread_fix_failures(void *);
static int      ufsfx_do_failure_q(void);
static void     ufsfx_kill_fix_failure_thread(void *);

/* routines called when failure occurs */
static int               ufs_fault_v(vnode_t *, char *, va_list)
        __KVPRINTFLIKE(2);
static ufs_failure_t    *init_failure(vnode_t *, char *, va_list)
        __KVPRINTFLIKE(2);
static void              queue_failure(ufs_failure_t *);
/*PRINTFLIKE2*/
static void              real_panic(ufs_failure_t *, const char *, ...)
        __KPRINTFLIKE(2);
static void              real_panic_v(ufs_failure_t *, const char *, va_list)
        __KVPRINTFLIKE(2);
static triage_t          triage(vnode_t *);

/* routines called when failure record is acted upon */
static sfrc_t   set_state(ufs_failure_t *, ufs_failure_states_t);
static int      state_trans_valid(ufs_failure_states_t, ufs_failure_states_t);
static int      terminal_state(ufs_failure_states_t);

/* routines called when states entered/found */
static sfrc_t   sf_minimum(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_undef(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_init(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_queue(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_found_queue(ufs_failure_t *);
static sfrc_t   sf_nonterm_cmn(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_term_cmn(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_panic(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_set_trylck(ufs_failure_t *);
static sfrc_t   sf_set_locked(ufs_failure_t *);
static sfrc_t   sf_found_trylck(ufs_failure_t *);
static sfrc_t   sf_found_lock_fix_cmn(ufs_failure_t *, ufs_failure_states_t);
static sfrc_t   sf_found_umount(ufs_failure_t *);

/* support routines, called by sf_nonterm_cmn and sf_term_cmn */
static time_t   trylock_time_exceeded(ufs_failure_t *);
static void     pester_msg(ufs_failure_t *, int);
static int      get_lockfs_status(ufs_failure_t *, struct lockfs *);
static void     alloc_lockfs_comment(ufs_failure_t *, struct lockfs *);
static int      set_lockfs(ufs_failure_t *, struct lockfs *);
static int      lockfs_failure(ufs_failure_t *);
static int      lockfs_success(ufs_failure_t *);
static int      fsck_active(ufs_failure_t *);

/* low-level support routines */
static ufsd_t   *get_state_desc(ufs_failure_states_t);
static char     *fs_name(ufs_failure_t *);

#if defined(DEBUG)
static char     *state_name(ufs_failure_states_t);
static char     *lock_name(struct lockfs *);
static char     *err_name(int);
static char     *act_name(ufsa_t);
static void      dump_uf_list(char *msg);
static void      dump_uf(ufs_failure_t *, int i);
#endif /* DEBUG */
/*
 *
 * State Transitions:
 *
 * normally:
 * if flagged to be locked but not unmounted:   (UFSMNT_ONERROR_LOCK)
 *      UNDEF -> INIT -> QUEUE -> TRYLCK -> LOCKED -> FIXING -> FIXED
 *
 * The only difference between these two is that the fsck must be started
 * manually.
 *
 * if flagged to be unmounted:                  (UFSMNT_ONERROR_UMOUNT)
 *      UNDEF -> INIT -> QUEUE -> TRYLCK -> LOCKED -> UMOUNT -> NOTFIX
 *
 * if flagged to panic:                         (UFSMNT_ONERROR_PANIC)
 *      UNDEF -> INIT -> PANIC
 *
 * if a secondary panic on a file system which has an active failure
 * record:
 *      UNDEF -> INIT -> QUEUE -> REPLICA
 *
 * UNDEF, INIT, QUEUE all are set in the context of the failing thread.
 * All other states (except possibly PANIC) are set in by the monitor
 * (lock) thread.
 *
 */

ufsd_t  state_desc[] =
{
        { UF_ILLEGAL,   "in an unknown state",  sf_minimum,     UF_ILLEGAL,
                                                                { 0, 1, 0 } },
        { UF_UNDEF,     "undefined",            sf_undef,       UF_UNDEF,
                                                                { 0, 1, 0 } },
        { UF_INIT,      "being initialized",    sf_init,        UF_UNDEF,
                                                                { 0, 1, 0 } },
        { UF_QUEUE,     "queued",               sf_queue,       UF_INIT,
                                                                { 0, 1, 0 } },
        { UF_TRYLCK,    "trying to be locked",  sf_nonterm_cmn,
                                                UF_QUEUE,       { 0, 0, 0 } },
        { UF_LOCKED,    "locked",               sf_nonterm_cmn,
                                        UF_TRYLCK | UF_FIXING,  { 0, 0, 0 } },
        { UF_UMOUNT,    "being unmounted",      sf_nonterm_cmn,

#if defined(DEBUG)
                                        UF_PANIC |
#endif /* DEBUG */
                                        UF_TRYLCK | UF_LOCKED,  { 0, 0, 0 } },
        { UF_FIXING,    "being fixed",          sf_nonterm_cmn,
                                                UF_LOCKED,      { 0, 0, 0 } },
        { UF_FIXED,     "fixed",                sf_term_cmn,
                                                UF_FIXING,      { 1, 0, 0 } },
        { UF_NOTFIX,    "not fixed",            sf_term_cmn,

#if defined(DEBUG)
                                                        UF_PANIC |
#endif /* DEBUG */

            UF_QUEUE | UF_TRYLCK | UF_LOCKED | UF_UMOUNT | UF_FIXING,
                                                                { 1, 0, 0 } },
        { UF_REPLICA,   "a replica",            sf_term_cmn,
                                                UF_QUEUE,       { 1, 0, 0 } },
        { UF_PANIC,     "panicking",            sf_panic,
                /* XXX make this narrower */    UF_ALLSTATES,   { 0, 0, 0 } },
        { UF_UNDEF,     NULL,                   ((sfrc_t (*)()) NULL),
                                                UF_UNDEF,       { 0, 0, 0 } }
};

/* unified collection */
struct ufsfx_info {
        struct uf_statistics            *ufi_statp;
        struct ufs_failure_tunable      *ufi_tunep;
        ufsd_t                          *ufi_statetab;
} uffsinfo;

#if defined(DEBUG)
struct action_description {
        ufsa_t   ad_v;
        char    *ad_name;
};

#define EUNK            (-1)

struct error_description {
        int      ed_errno;
        char    *ed_name;
} err_desc[] =
{
        { EUNK,         "<unexpected errno?>"   },
        { EINVAL,       "EINVAL"                },
        { EACCES,       "EACCES"                },
        { EPERM,        "EPERM"                 },
        { EIO,          "EIO"                   },
        { EDEADLK,      "EDEADLK"               },
        { EBUSY,        "EBUSY"                 },
        { EAGAIN,       "EAGAIN"                },
        { ERESTART,     "ERESTART"              },
        { ETIMEDOUT,    "ETIMEDOUT"             },
        { NO_ERROR,     "Ok"                    },
        { EUNK,         NULL                    }
};

struct action_description act_desc[] =
{
        { UFA_ERROR,    "<unexpected action?>"  },
        { UFA_FOUND,    "\"found\""     },
        { UFA_SET,      "\"set\""       },
        { UFA_ERROR,    NULL                    },
};

#define LOCKFS_BADLOCK  (-1)

struct lock_description {
        int      ld_type;
        char    *ld_name;
} lock_desc[] =
{
        { LOCKFS_BADLOCK,       "<unexpected lock?>"    },
        { LOCKFS_ULOCK,         "Unlock"                },
        { LOCKFS_ELOCK,         "Error Lock"            },
        { LOCKFS_HLOCK,         "Hard Lock"             },
        { LOCKFS_OLOCK,         "Old Lock"              },
        { LOCKFS_BADLOCK,       NULL                    }
};

#endif /* DEBUG */

/*
 * ufs_fault, ufs_fault_v
 *
 *  called instead of cmn_err(CE_PANIC, ...) by ufs routines
 *  when a failure is detected to put the file system into an
 *  error state (if possible) or to devolve to a panic otherwise
 *
 * vnode is some vnode in this file system, used to find the way
 * to ufsvfs, vfsp etc.  Since a panic can be called from many
 * levels, the vnode is the most convenient hook to pass through.
 *
 */

/*PRINTFLIKE2*/
int
ufs_fault(vnode_t *vp, char *fmt, ...)
{
        va_list adx;
        int     error;

        MINOR(("[ufs_fault"));

        va_start(adx, fmt);
        error = ufs_fault_v(vp, fmt, adx);
        va_end(adx);

        MINOR((": %s (%d)]\n", err_name(error), error));
        return (error);
}

const char *nullfmt = "<null format?>";

static int
ufs_fault_v(vnode_t *vp, char *fmt, va_list adx)
{
        ufs_failure_t           *new = NULL;
        ufsvfs_t                *ufsvfsp;
        triage_t                 fix;
        int                      err = ERESTART;
        int                     need_vfslock;

        MINOR(("[ufs_fault_v"));

        if (fmt == NULL)
                fmt = (char *)nullfmt;

        fix = triage(vp);

        if (vp) {
                ufsvfsp = (struct ufsvfs *)vp->v_vfsp->vfs_data;

                /*
                 * Something bad has happened. That is why we are here.
                 *
                 * In order for the bad thing to be recorded in the superblock
                 * we need to write to the superblock directly.
                 * In the case that logging is enabled the logging code
                 * would normally intercept our write as a delta to the log,
                 * thus we mark the filesystem FSBAD in any case.
                 */
                need_vfslock = !MUTEX_HELD(&ufsvfsp->vfs_lock);

                if (need_vfslock) {
                        mutex_enter(&ufsvfsp->vfs_lock);
                }

                ufsvfsp->vfs_fs->fs_clean = FSBAD;
                ASSERT(SEMA_HELD(&ufsvfsp->vfs_bufp->b_sem));
                ufsvfsp->vfs_bufp->b_flags &=
                    ~(B_ASYNC | B_READ | B_DONE | B_ERROR | B_DELWRI);

                (void) bdev_strategy(ufsvfsp->vfs_bufp);
                (void) biowait(ufsvfsp->vfs_bufp);

                if (need_vfslock) {
                        mutex_exit(&ufsvfsp->vfs_lock);
                }
        }

        switch (fix) {

        default:
        case TRIAGE_DEAD:
        case TRIAGE_NO_SPIRIT:

                real_panic_v(new, fmt, adx);
                /* LINTED: warning: logical expression always true: op "||" */
                ASSERT(DEBUG);
                err = EAGAIN;

#if defined(DEBUG)
                if (!(DEBUG_FLAGS & DBGFLG_FIXWOULDPANIC)) {
                        break;
                }
#else
                break;

#endif /* DEBUG */
                /* FALLTHROUGH */

        case TRIAGE_ATTEND_TO:

                /* q thread not running yet? */
                if (mutex_tryenter(&ufs_fix.uq_mutex)) {
                        if (!ufs_fix.uq_threadp) {
                                mutex_exit(&ufs_fix.uq_mutex);
                                ufs_thread_start(&ufs_fix,
                                    ufsfx_thread_fix_failures, NULL);
                                ufs_fix.uq_threadp->t_flag |= T_DONTBLOCK;
                                mutex_enter(&ufs_fix.uq_mutex);
                        } else {
                                /*
                                 * We got the lock but we are not the current
                                 * threadp so we have to release the lock.
                                 */
                                mutex_exit(&ufs_fix.uq_mutex);
                        }
                } else {
                        MINOR((": fix failure thread already running "));
                        /*
                         * No need to log another failure as one is already
                         * being logged.
                         */
                        break;
                }

                if (ufs_fix.uq_threadp && ufs_fix.uq_threadp == curthread) {
                        mutex_exit(&ufs_fix.uq_mutex);
                        cmn_err(CE_WARN, "ufs_fault_v: recursive ufs_fault");
                } else {
                        /*
                         * Must check if we actually still own the lock and
                         * if so then release the lock and move on with life.
                         */
                        if (mutex_owner(&ufs_fix.uq_mutex) == curthread)
                                mutex_exit(&ufs_fix.uq_mutex);
                }

                new = init_failure(vp, fmt, adx);
                if (new != NULL) {
                        queue_failure(new);
                        break;
                }
                real_panic_v(new, fmt, adx);
                break;

        }
        MINOR(("] "));
        return (err);
}

/*
 * triage()
 *
 *  Attempt to fix iff:
 *    - the system is not already panicking
 *    - this file system isn't explicitly marked not to be fixed
 *    - we can connect to the user-level daemon
 * These conditions are detectable later, but if we can determine
 * them in the failing threads context the core dump may be more
 * useful.
 *
 */

static triage_t
triage(vnode_t *vp)
{
        struct inode     *ip;
        int               need_unlock_vfs;
        int               fs_flags;

        MINUTE(("[triage"));

        if (panicstr) {
                MINUTE((
                ": already panicking: \"%s\" => TRIAGE_DEAD]\n", panicstr));
                return (TRIAGE_DEAD);
        }

        if (!vp || !(ip = VTOI(vp)) || !ip->i_ufsvfs) {
                MINUTE((
        ": vp, ip or ufsvfs is NULL; can't determine fs => TRIAGE_DEAD]\n"));
                return (TRIAGE_DEAD);
        }

        /* use tryenter and continue no matter what since we're panicky */
        need_unlock_vfs = !MUTEX_HELD(&ip->i_ufsvfs->vfs_lock);
        if (need_unlock_vfs)
                need_unlock_vfs = mutex_tryenter(&ip->i_ufsvfs->vfs_lock);

        fs_flags = ip->i_ufsvfs->vfs_fsfx.fx_flags;
        if (need_unlock_vfs)
                mutex_exit(&ip->i_ufsvfs->vfs_lock);

        if (fs_flags & UFSFX_PANIC) {
                MINUTE((
                ": filesystem marked \"panic\" => TRIAGE_NO_SPIRIT]\n"));
                return (TRIAGE_NO_SPIRIT);
        }

        if (ufs_checkaccton(vp) != 0) {
                MINUTE((
                ": filesystem would deadlock (accounting) => TRIAGE_DEAD]\n"));
                return (TRIAGE_DEAD);
        }

        if (ufs_checkswapon(vp) != 0) {
                MINUTE((
                ": filesystem would deadlock (swapping) => TRIAGE_DEAD]\n"));
                return (TRIAGE_DEAD);
        }

        MINUTE((": return TRIAGE_ATTEND_TO] "));
        return (TRIAGE_ATTEND_TO);
}

/*
 * init failure
 *
 * This routine allocates a failure struct and initializes
 * it's member elements.
 * Space is allocated for copies of dynamic identifying fs structures
 * passed in.  Without a much more segmented kernel architecture
 * this is as protected as we can make it (for now.)
 */
static ufs_failure_t *
init_failure(vnode_t *vp, char *fmt, va_list adx)
{
        ufs_failure_t   *new;
        struct inode    *ip;
        int              initialization_worked = 0;
        int              need_vfs_unlock;

        MINOR(("[init_failure"));

        new = kmem_zalloc(sizeof (ufs_failure_t), KM_NOSLEEP);
        if (!new) {
                MINOR((": kmem_zalloc failed]\n"));
                return (NULL);
        }

        /*
         * enough information to make a fix attempt possible?
         */
        if (!vp || !(ip = VTOI(vp)) || !ip->i_ufsvfs || !vp->v_vfsp ||
            !ip->i_ufsvfs->vfs_bufp || !ITOF(ip) || !fmt)
                goto errout;

        if (vp->v_type != VREG && vp->v_type != VDIR &&
            vp->v_type != VBLK && vp->v_type != VCHR &&
            vp->v_type != VLNK && vp->v_type != VFIFO &&
            vp->v_type != VSOCK)
                goto errout;

        if (ip->i_ufsvfs->vfs_root->v_type != VREG &&
            ip->i_ufsvfs->vfs_root->v_type != VDIR &&
            ip->i_ufsvfs->vfs_root->v_type != VBLK &&
            ip->i_ufsvfs->vfs_root->v_type != VCHR &&
            ip->i_ufsvfs->vfs_root->v_type != VLNK &&
            ip->i_ufsvfs->vfs_root->v_type != VFIFO &&
            ip->i_ufsvfs->vfs_root->v_type != VSOCK)
                goto errout;

        if ((ITOF(ip)->fs_magic != FS_MAGIC) &&
            (ITOF(ip)->fs_magic != MTB_UFS_MAGIC))
                goto errout;

        /* intialize values */

        (void) vsnprintf(new->uf_panic_str, LOCKFS_MAXCOMMENTLEN - 1, fmt, adx);

        new->uf_ufsvfsp = ip->i_ufsvfs;
        new->uf_vfsp    = ip->i_vfs;

        mutex_init(&new->uf_mutex, NULL, MUTEX_DEFAULT, NULL);
        need_vfs_unlock = !MUTEX_HELD(&ip->i_ufsvfs->vfs_lock);

        if (need_vfs_unlock) {
                if (!mutex_tryenter(&ip->i_ufsvfs->vfs_lock)) {
                        /*
                         * not much alternative here, but we're panicking
                         * already, it couldn't be worse - so just
                         * proceed optimistically and take note.
                         */
                        mutex_enter(&uf_stats.ufst_mutex);
                        uf_stats.ufst_lock_violations++;
                        mutex_exit(&uf_stats.ufst_mutex);
                        MINOR((": couldn't get vfs lock"))
                        need_vfs_unlock = 0;
                }
        }

        if (mutex_tryenter(&new->uf_mutex)) {
                initialization_worked = set_state(new, UF_INIT);
                mutex_exit(&new->uf_mutex);
        }

        if (need_vfs_unlock)
                mutex_exit(&ip->i_ufsvfs->vfs_lock);

        if (initialization_worked) {
                MINOR(("] "));
                return (new);
        }
        /* FALLTHROUGH */

errout:
        if (new)
                kmem_free(new, sizeof (ufs_failure_t));
        MINOR((": failed]\n"));
        return (NULL);
}

static void
queue_failure(ufs_failure_t *new)
{
        MINOR(("[queue_failure"));

        mutex_enter(&ufs_fix.uq_mutex);

        if (ufs_fix.uq_ufhead)
                insque(new, &ufs_fix.uq_ufhead);
        else
                ufs_fix.uq_ufhead = new;

        if (mutex_tryenter(&new->uf_mutex)) {
                (void) set_state(new, UF_QUEUE);
                mutex_exit(&new->uf_mutex);
        }

        mutex_enter(&uf_stats.ufst_mutex);              /* force wakeup */
        ufs_fix.uq_ne = ufs_fix.uq_lowat = uf_stats.ufst_num_failed;
        mutex_exit(&uf_stats.ufst_mutex);

        cv_broadcast(&ufs_fix.uq_cv);

        DCALL(DBGLVL_MAJOR, cmn_err(CE_WARN, new->uf_panic_str ?
            new->uf_panic_str : "queue_failure: NULL panic str?"));
        mutex_exit(&ufs_fix.uq_mutex);

        MINOR(("] "));
}

/*PRINTFLIKE2*/
static void
real_panic(ufs_failure_t *f, const char *fmt, ...)
{
        va_list adx;

        MINUTE(("[real_panic "));

        va_start(adx, fmt);
        real_panic_v(f, fmt, adx);
        va_end(adx);

        MINUTE((": return?!]\n"));
}

static void
real_panic_v(ufs_failure_t *f, const char *fmt, va_list adx)
{
        int seriousness = CE_PANIC;
        int need_unlock;

        MINUTE(("[real_panic_v "));

        if (f && f->uf_ufsvfsp)
                TRANS_SETERROR(f->uf_ufsvfsp);

#if defined(DEBUG)
        if (DEBUG_FLAGS & DBGFLG_NOPANIC) {
                seriousness = CE_WARN;
                cmn_err(CE_WARN, "real_panic: EWOULDPANIC\n");
        }
#endif /* DEBUG */

        delay(hz >> 1);                 /* allow previous warnings to get out */

        if (!f && fmt)
                vcmn_err(seriousness, fmt, adx);
        else
                cmn_err(seriousness, f != NULL && f->uf_panic_str[0] != '\0' ?
                    f->uf_panic_str: "real_panic: <unknown panic?>");

        if (f) {
                need_unlock = !MUTEX_HELD(&f->uf_mutex);
                if (need_unlock) {
                        mutex_enter(&f->uf_mutex);
                }

                f->uf_retry = -1;
                (void) set_state(f, UF_PANIC);

                if (need_unlock) {
                        mutex_exit(&f->uf_mutex);
                }
        }
        MINUTE((": return?!]\n"));
}

/*
 * initializes ufs panic structs, locks, etc
 */
void
ufsfx_init(void)
{

        MINUTE(("[ufsfx_init"));

        /* patchable; unchanged while running, so no lock is needed */
        ufsfx_tune.uft_too_long         = UF_TOO_LONG;
        ufsfx_tune.uft_fixstart_period  = UF_FIXSTART_PERIOD;
        ufsfx_tune.uft_fixpoll_period   = UF_FIXPOLL_PERIOD;
        ufsfx_tune.uft_short_err_period = UF_SHORT_ERROR_PERIOD;
        ufsfx_tune.uft_long_err_period  = UF_LONG_ERROR_PERIOD;

        uffsinfo.ufi_statp      = &uf_stats;
        uffsinfo.ufi_tunep      = &ufsfx_tune;
        uffsinfo.ufi_statetab   = &state_desc[0];

        mutex_init(&uf_stats.ufst_mutex, NULL, MUTEX_DEFAULT, NULL);
        ufs_thread_init(&ufs_fix, /* maxne */ 1);

        MINUTE(("] "));
}

/*
 * initializes per-ufs values
 * returns 0 (ok) or errno
 */
int
ufsfx_mount(struct ufsvfs *ufsvfsp, int flags)
{
        MINUTE(("[ufsfx_mount (%d)", flags));
        /* don't check/need vfs_lock because it's still being initialized */

        ufsvfsp->vfs_fsfx.fx_flags = (flags & UFSMNT_ONERROR_FLGMASK) >> 4;

        MINUTE((": %s: fx_flags:%ld,",
            ufsvfsp->vfs_fs->fs_fsmnt, ufsvfsp->vfs_fsfx.fx_flags));
        /*
         *      onerror={panic ^ lock only ^ unmount}
         */

        if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_PANIC) {
                MINUTE((" PANIC"));

        } else if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_LCKONLY) {
                MINUTE((" LCKONLY"));

        } else if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_LCKUMOUNT) {
                MINUTE((" LCKUMOUNT"));

        } else {
                ufsvfsp->vfs_fsfx.fx_flags = UFSFX_DEFAULT;
                ASSERT(ufsvfsp->vfs_fsfx.fx_flags &
                    (UFSMNT_ONERROR_FLGMASK >> 4));
                MINUTE((" DEFAULT"));
        }

        pollwakeup(&ufs_pollhd, POLLPRI);
        MINUTE(("]\n"));
        return (0);
}

/*
 * ufsfx_unmount
 *
 * called during unmount
 */
void
ufsfx_unmount(struct ufsvfs *ufsvfsp)
{
        ufs_failure_t   *f;
        int              must_unlock_list;

        MINUTE(("[ufsfx_unmount"));

        if (!ufsvfsp) {
                MINUTE((": no ufsvfsp]"));
                return;
        }

        if ((must_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex)) != 0)
                mutex_enter(&ufs_fix.uq_mutex);

        for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {
                int must_unlock_failure;

                must_unlock_failure = !MUTEX_HELD(&f->uf_mutex);
                if (must_unlock_failure) {
                        mutex_enter(&f->uf_mutex);
                }

                if (f->uf_ufsvfsp == ufsvfsp) {

                        /*
                         * if we owned the failure record lock, then this
                         * is probably a fix failure-triggered unmount, so
                         * the warning is not appropriate or needed
                         */

                        /* XXX if rebooting don't print this? */
                        if (!terminal_state(f->uf_s) && must_unlock_failure) {
                                cmn_err(CE_WARN,
                                    "Unmounting %s while error-locked",
                                    fs_name(f));
                        }

                        f->uf_ufsvfsp           = NULL;
                        f->uf_vfs_ufsfxp        = NULL;
                        f->uf_vfs_lockp         = NULL;
                        f->uf_bp                = NULL;
                        f->uf_vfsp              = NULL;
                        f->uf_retry             = -1;
                }

                if (must_unlock_failure)
                        mutex_exit(&f->uf_mutex);
        }
        if (must_unlock_list)
                mutex_exit(&ufs_fix.uq_mutex);

        pollwakeup(&ufs_pollhd, POLLPRI | POLLHUP);
        MINUTE(("] "));
}

/*
 * ufsfx_(un)lockfs
 *
 * provides hook from lockfs code so we can recognize unlock/relock
 *  This is called after it is certain that the (un)lock will succeed.
 */
void
ufsfx_unlockfs(struct ufsvfs *ufsvfsp)
{
        ufs_failure_t   *f;
        int              need_unlock;
        int              need_unlock_list;
        int              informed = 0;

        MINUTE(("[ufsfx_unlockfs"));

        if (!ufsvfsp)
                return;

        need_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex);

        if (need_unlock_list)
                mutex_enter(&ufs_fix.uq_mutex);

        for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

                need_unlock = !MUTEX_HELD(&f->uf_mutex);
                if (need_unlock)
                        mutex_enter(&f->uf_mutex);

                if (f->uf_ufsvfsp == ufsvfsp && !terminal_state(f->uf_s)) {
                        if (!(f->uf_s & UF_FIXING)) {
                                /*
                                 * This might happen if we don't notice that
                                 * the fs gets marked FSFIX before it is
                                 * marked FSCLEAN, as might occur if the
                                 * the superblock was hammered directly.
                                 */
                                if (!informed) {
                                        informed = 1;
                                        cmn_err(CE_NOTE,
                                            "Unlock of %s succeeded before "
                                            "fs_clean marked FSFIX?",
                                            fs_name(f));
                                }

                                /*
                                 * pass through fixing state so
                                 * transition protocol is satisfied
                                 */
                                if (!set_state(f, UF_FIXING)) {
                                        MINUTE((": failed] "));
                                }
                        }

                        if (!set_state(f, UF_FIXED)) {
                                /* it's already fixed, so don't panic now */
                                MINUTE((": failed] "));
                        }
                }

                if (need_unlock)
                        mutex_exit(&f->uf_mutex);
        }
        if (need_unlock_list)
                mutex_exit(&ufs_fix.uq_mutex);
        MINUTE(("] "));
}

void
ufsfx_lockfs(struct ufsvfs *ufsvfsp)
{
        ufs_failure_t   *f;
        int              need_unlock;
        int              need_unlock_list;

        MINUTE(("[ufsfx_lockfs"));

        if (!ufsvfsp)
                return;

        need_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex);

        if (need_unlock_list)
                mutex_enter(&ufs_fix.uq_mutex);

        for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

                need_unlock = !MUTEX_HELD(&f->uf_mutex);
                if (need_unlock)
                        mutex_enter(&f->uf_mutex);

                if (f->uf_ufsvfsp == ufsvfsp && !terminal_state(f->uf_s) &&
                    f->uf_s != UF_PANIC) {
                        switch (f->uf_s) {

                        default:
                                cmn_err(CE_WARN,
                                    "fs %s not in state "
                                    "UF_TRYLCK, UF_LOCKED or UF_FIXING",
                                    fs_name(f));
                                break;

                        case UF_TRYLCK:
                                if (!set_state(f, UF_LOCKED)) {
                                        MINUTE((": failed] "));
                                }
                                break;

                        case UF_LOCKED:
                                if (!set_state(f, UF_FIXING)) {
                                        MINUTE((": failed] "));
                                }
                                break;

                        case UF_FIXING:
                                break;

                        }
                }

                if (need_unlock)
                        mutex_exit(&f->uf_mutex);
        }
        if (need_unlock_list)
                mutex_exit(&ufs_fix.uq_mutex);

        MINUTE(("] "));
}

/*
 * error lock, trigger fsck and unlock those fs with failures
 * blatantly copied from the hlock routine, although this routine
 * triggers differently in order to use uq_ne as meaningful data.
 */
/* ARGSUSED */
void
ufsfx_thread_fix_failures(void *ignored)
{
        int             retry;
        callb_cpr_t     cprinfo;

        CALLB_CPR_INIT(&cprinfo, &ufs_fix.uq_mutex, callb_generic_cpr,
            "ufsfixfail");

        MINUTE(("[ufsfx_thread_fix_failures] "));

        for (;;) {
                /* sleep until there is work to do */

                mutex_enter(&ufs_fix.uq_mutex);
                (void) ufs_thread_run(&ufs_fix, &cprinfo);
                ufs_fix.uq_ne = 0;
                mutex_exit(&ufs_fix.uq_mutex);

                /* process failures on our q */
                do {
                        retry = ufsfx_do_failure_q();
                        if (retry) {
                                mutex_enter(&ufs_fix.uq_mutex);
                                CALLB_CPR_SAFE_BEGIN(&cprinfo);
                                (void) cv_reltimedwait(&ufs_fix.uq_cv,
                                    &ufs_fix.uq_mutex, (hz * retry),
                                    TR_CLOCK_TICK);
                                CALLB_CPR_SAFE_END(&cprinfo,
                                    &ufs_fix.uq_mutex);
                                mutex_exit(&ufs_fix.uq_mutex);
                        }
                } while (retry);
        }
        /* NOTREACHED */
}


/*
 * watch for fix-on-panic work
 *
 * returns # of seconds to sleep before trying again
 * and zero if no retry is needed
 */

int
ufsfx_do_failure_q(void)
{
        ufs_failure_t   *f;
        long             retry = 1;
        ufsd_t          *s;

        MAJOR(("[ufsfx_do_failure_q"));
        DCALL(DBGLVL_HIDEOUS, dump_uf_list(NULL));

        if (!mutex_tryenter(&ufs_fix.uq_mutex))
                return (retry);

        retry = 0;
rescan_q:

        /*
         * walk down failure list
         *  depending on state of each failure, do whatever
         *  is appropriate to move it to the next state
         *  taking note of whether retry gets set
         *
         * retry protocol:
         * wakeup in shortest required time for any failure
         *   retry == 0; nothing more to do (terminal state)
         *   retry < 0; reprocess queue immediately, retry will
         *              be abs(retry) for the next cycle
         *   retry > 0; schedule wakeup for retry seconds
         */

        for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

                if (!mutex_tryenter(&f->uf_mutex)) {
                        retry = 1;
                        continue;
                }
                s = get_state_desc(f->uf_s);

                MINOR((": found%s: %s, \"%s: %s\"\n",
                    s->ud_attr.terminal ? " old" : "",
                    fs_name(f), state_name(f->uf_s), f->uf_panic_str));

                if (s->ud_attr.terminal) {
                        mutex_exit(&f->uf_mutex);
                        continue;
                }

                if (s->ud_sfp)
                        (*s->ud_sfp)(f, UFA_FOUND, f->uf_s);

                ASSERT(terminal_state(f->uf_s) || f->uf_retry != 0);

                if (f->uf_retry != 0) {
                        if (retry > f->uf_retry || retry == 0)
                                retry = f->uf_retry;
                        if (f->uf_retry < 0)
                                f->uf_retry = abs(f->uf_retry);
                }
                mutex_exit(&f->uf_mutex);
        }


        if (retry < 0) {
                retry = abs(retry);
                goto rescan_q;
        }

        mutex_exit(&ufs_fix.uq_mutex);

        DCALL(DBGLVL_HIDEOUS, dump_uf_list(NULL));
        MAJOR((": retry=%ld, good night]\n\n", retry));

        return (retry);
}

static void
pester_msg(ufs_failure_t *f, int seriousness)
{
        MINUTE(("[pester_msg"));
        ASSERT(f->uf_s & (UF_LOCKED | UF_FIXING));

        /*
         * XXX if seems too long for this fs, poke administrator
         * XXX to run fsck manually (and change retry time?)
         */
        cmn_err(seriousness, "Waiting for repair of %s to %s",
            fs_name(f), f->uf_s & UF_LOCKED ? "start" : "finish");
        MINUTE(("]"));
}

static time_t
trylock_time_exceeded(ufs_failure_t *f)
{
        time_t          toolong;
        extern time_t   time;

        MINUTE(("[trylock_time_exceeded"));
        ASSERT(MUTEX_HELD(&f->uf_mutex));

        toolong = (time_t)ufsfx_tune.uft_too_long + f->uf_entered_tm;
        if (time > toolong)
                cmn_err(CE_WARN, "error-lock timeout exceeded: %s", fs_name(f));

        MINUTE(("] "));
        return (time <= toolong? 0: time - toolong);
}

static int
get_lockfs_status(ufs_failure_t *f, struct lockfs *lfp)
{
        MINUTE(("[get_lockfs_status"));

        if (!f->uf_ufsvfsp) {
                MINUTE((": ufsvfsp is NULL]\n"));
                return (0);
        }

        ASSERT(MUTEX_HELD(&f->uf_mutex));
        ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
        ASSERT(!vfs_lock_held(f->uf_vfsp));
        ASSERT(f->uf_ufsvfsp->vfs_root != NULL);

        f->uf_lf_err = ufs_fiolfss(f->uf_ufsvfsp->vfs_root, lfp);

        if (f->uf_lf_err) {
                f->uf_retry = ufsfx_tune.uft_short_err_period;
        }

        MINUTE(("] "));
        return (1);
}

static sfrc_t
set_state(ufs_failure_t *f, ufs_failure_states_t new_state)
{
        ufsd_t          *s;
        sfrc_t           sfrc = SFRC_FAIL;
        int              need_unlock;
        extern time_t    time;

        HIDEOUS(("[set_state: new state:%s", state_name(new_state)));
        ASSERT(f);
        ASSERT(MUTEX_HELD(&f->uf_mutex));

        /*
         * if someone else is panicking, just let panic sync proceed
         */
        if (panicstr) {
                (void) set_state(f, UF_NOTFIX);
                HIDEOUS((": state reset: not fixed] "));
                return (sfrc);
        }

        /*
         * bad state transition, an internal error
         */
        if (!state_trans_valid(f->uf_s, new_state)) {
                /* recursion */
                if (!(f->uf_s & UF_PANIC) && !(new_state & UF_PANIC))
                        (void) set_state(f, UF_PANIC);
                MINOR((": state reset: transition failure (\"%s\"->\"%s\")] ",
                    state_name(f->uf_s), state_name(new_state)));
                return (sfrc);
        }

        s = get_state_desc(new_state);

        need_unlock = !MUTEX_HELD(&ufs_fix.uq_mutex);
        if (need_unlock)
                mutex_enter(&ufs_fix.uq_mutex);

        if (s->ud_attr.at_fail && ufs_fix.uq_threadp &&
            curthread == ufs_fix.uq_threadp) {
                cmn_err(CE_WARN, "set_state: probable recursive panic of %s",
                    fs_name(f));
        }
        if (need_unlock)
                mutex_exit(&ufs_fix.uq_mutex);

        /* NULL state functions always succeed */
        sfrc = !s->ud_sfp? SFRC_SUCCESS: (*s->ud_sfp)(f, UFA_SET, new_state);

        if (sfrc == SFRC_SUCCESS && f->uf_s != new_state) {
                f->uf_s = new_state;
                f->uf_entered_tm = time;
                f->uf_counter = 0;
        }

        HIDEOUS(("]\n"));
        return (sfrc);
}

static ufsd_t *
get_state_desc(ufs_failure_states_t state)
{
        ufsd_t *s;

        HIDEOUS(("[get_state_desc"));

        for (s = &state_desc[1]; s->ud_name != NULL; s++) {
                if (s->ud_v == state) {
                        HIDEOUS(("] "));
                        return (s);
                }
        }

        HIDEOUS(("] "));
        return (&state_desc[0]);        /* default */
}

static sfrc_t
sf_undef(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
{
        sfrc_t rc;

        TRIVIA(("[sf_undef, action is %s, state is %s\n",
            act_name(a), state_name(s)));
        ASSERT(s == UF_UNDEF);

        /* shouldn't find null failure records or ever set one */
        rc = set_state(f, UF_NOTFIX);

        TRIVIA(("] "));
        return (rc);
}


static sfrc_t
sf_init(
        ufs_failure_t   *f,
        ufsa_t   a,
        ufs_failure_states_t     s)
{
        sfrc_t          rc = SFRC_FAIL;
        extern time_t   time;

        TRIVIA(("[sf_init, action is %s", act_name(a)));
        ASSERT(s & UF_INIT);

        switch (a) {
        case UFA_SET:
                f->uf_begin_tm = time;
                f->uf_retry = 1;
                if (!f->uf_ufsvfsp) {
                        (void) set_state(f, UF_PANIC);
                        TRIVIA((": NULL ufsvfsp]\n"));
                        return (rc);
                }
                /*
                 * because we can call panic from many different levels,
                 * we can't be sure that we've got the vfs_lock at this
                 * point.  However, there's not much alternative and if
                 * we don't (have the lock) the worst case is we'll just
                 * panic again
                 */
                f->uf_vfs_lockp         = &f->uf_ufsvfsp->vfs_lock;
                f->uf_vfs_ufsfxp        = &f->uf_ufsvfsp->vfs_fsfx;

                if (!f->uf_ufsvfsp->vfs_bufp) {
                        (void) set_state(f, UF_PANIC);
                        TRIVIA((": NULL vfs_bufp]\n"));
                        return (rc);
                }
                f->uf_bp = f->uf_ufsvfsp->vfs_bufp;

                if (!f->uf_ufsvfsp->vfs_bufp->b_un.b_fs) {
                        (void) set_state(f, UF_PANIC);
                        TRIVIA((": NULL vfs_fs]\n"));
                        return (rc);
                }

                /* vfs_fs = vfs_bufp->b_un.b_fs */
                bcopy(f->uf_ufsvfsp->vfs_fs->fs_fsmnt, f->uf_fsname, MAXMNTLEN);

                f->uf_lf.lf_lock  = LOCKFS_ELOCK;       /* primer */

                if (!f->uf_vfsp || f->uf_vfsp->vfs_dev == NODEV) {
                        (void) set_state(f, UF_PANIC);
                        TRIVIA((": NULL vfsp or vfs_dev == NODEV"));
                        return (rc);
                }
                f->uf_dev = f->uf_vfsp->vfs_dev;

                rc = SFRC_SUCCESS;
                break;

        case UFA_FOUND:
        default:
                /* failures marked init shouldn't even be on the queue yet */
                rc = set_state(f, UF_QUEUE);
                TRIVIA((": found failure with state init]\n"));
        }

        TRIVIA(("] "));
        return (rc);
}

static sfrc_t
sf_queue(
        ufs_failure_t   *f,
        ufsa_t   a,
        ufs_failure_states_t     s)
{
        sfrc_t          rc = SFRC_FAIL;

        TRIVIA(("[sf_queue, action is %s", act_name(a)));
        ASSERT(s & UF_QUEUE);

        if (!f->uf_ufsvfsp) {
                TRIVIA((": NULL ufsvfsp]\n"));
                return (rc);
        }

        switch (a) {
        case UFA_FOUND:
                rc = sf_found_queue(f);
                break;

        case UFA_SET:

                ASSERT(MUTEX_HELD(&ufs_fix.uq_mutex));

                mutex_enter(&uf_stats.ufst_mutex);
                uf_stats.ufst_num_failed++;
                mutex_exit(&uf_stats.ufst_mutex);

                /*
                 * if can't get the vfs lock, just wait until
                 * UF_TRYLCK to set fx_current
                 */
                if (mutex_tryenter(f->uf_vfs_lockp)) {
                        f->uf_vfs_ufsfxp->fx_current = f;
                        mutex_exit(f->uf_vfs_lockp);
                } else {
                        mutex_enter(&uf_stats.ufst_mutex);
                        uf_stats.ufst_current_races++;
                        mutex_exit(&uf_stats.ufst_mutex);
                }

                f->uf_retry = 1;
                rc = SFRC_SUCCESS;
                TRIVIA(("] "));
                break;

        default:
                (void) set_state(f, UF_PANIC);
                TRIVIA((": failed] "));
        }

        return (rc);
}

static sfrc_t
sf_found_queue(ufs_failure_t *f)
{
        int             replica;
        sfrc_t          rc = SFRC_FAIL;

        TRIVIA(("[sf_found_queue"));

        /*
         * don't need to check for null ufsvfsp because
         * unmount must own list's ufs_fix.uq_mutex
         * to mark it null and we own that lock since
         * we got here.
         */

        ASSERT(MUTEX_HELD(&ufs_fix.uq_mutex));
        ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));

        if (!mutex_tryenter(f->uf_vfs_lockp)) {
                TRIVIA((": tryenter(vfslockp) failed; retry]\n"));
                f->uf_retry = 1;
                return (rc);
        }

        replica = f->uf_vfs_ufsfxp && f->uf_vfs_ufsfxp->fx_current != NULL &&
            f->uf_vfs_ufsfxp->fx_current != f &&
            !terminal_state(f->uf_vfs_ufsfxp->fx_current->uf_s);

        /*
         * copy general flags to this ufs_failure so we don't
         * need to refer back to the ufsvfs, or, more importantly,
         * don't need to keep acquiring (trying to acquire) vfs_lockp
         *
         * The most restrictive option wins:
         *  panic > errlock only > errlock+unmount > repair
         * XXX panic > elock > elock > elock+umount
         */
        if (f->uf_vfs_ufsfxp->fx_flags & UFSFX_PANIC) {
                if (!set_state(f, UF_PANIC)) {
                        TRIVIA((": marked panic but was queued?"));
                        real_panic(f, " ");
                        /*NOTREACHED*/
                }
                mutex_exit(f->uf_vfs_lockp);
                return (rc);
        }
        f->uf_flags = f->uf_vfs_ufsfxp->fx_flags;

        if (replica) {
                if (!set_state(f, UF_REPLICA)) {
                        f->uf_retry = 1;
                        TRIVIA((": set to replica failed] "));
                } else {
                        TRIVIA(("] "));
                }
                mutex_exit(f->uf_vfs_lockp);
                return (rc);
        }
        mutex_exit(f->uf_vfs_lockp);

        if (!set_state(f, UF_TRYLCK)) {
                TRIVIA((": failed] "));
        } else {
                rc = SFRC_SUCCESS;
        }
        return (rc);
}

static sfrc_t
sf_nonterm_cmn(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
{
        sfrc_t  rc = SFRC_FAIL;

        TRIVIA(("[sf_nonterm_cmn, action: %s, %s", act_name(a), state_name(s)));
        ASSERT(s & (UF_TRYLCK | UF_LOCKED | UF_UMOUNT | UF_FIXING));
        ASSERT(!terminal_state(s));

        if (!f->uf_ufsvfsp && !(f->uf_s & UF_UMOUNT)) {
                TRIVIA((": NULL ufsvfsp (state != UMOUNT)]\n"));
                (void) set_state(f, UF_NOTFIX);
                return (rc);
        }

        switch (a) {
        case UFA_SET:
                switch (s) {
                case UF_TRYLCK:
                        ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
                        rc = sf_set_trylck(f);
                        break;

                case UF_LOCKED:
                        rc = sf_set_locked(f);
                        break;

                case UF_FIXING:
                        f->uf_flags |= UFSFX_REPAIR_START;
                        f->uf_retry  = ufsfx_tune.uft_fixpoll_period;
                        rc = SFRC_SUCCESS;
                        break;

                case UF_UMOUNT:
                        f->uf_retry = -ufsfx_tune.uft_short_err_period;
                        rc = SFRC_SUCCESS;
                        break;

                default:
                        (void) set_state(f, UF_PANIC);
                        TRIVIA((": failed] "));
                }
                break;

        case UFA_FOUND:

                switch (s) {
                case UF_TRYLCK:
                        rc = sf_found_trylck(f);
                        break;

                case UF_LOCKED:
                case UF_FIXING:
                        rc = sf_found_lock_fix_cmn(f, s);
                        break;

                case UF_UMOUNT:
                        rc = sf_found_umount(f);
                        break;

                default:
                        (void) set_state(f, UF_PANIC);
                        TRIVIA((": failed] "));
                        break;
                }
                break;
        default:
                (void) set_state(f, UF_PANIC);
                TRIVIA((": failed] "));
                break;
        }

        TRIVIA(("] "));
        return (rc);
}

static sfrc_t
sf_set_trylck(ufs_failure_t *f)
{
        TRIVIA(("[sf_set_trylck"));

        if (!mutex_tryenter(f->uf_vfs_lockp)) {
                TRIVIA((": tryenter(vfslockp) failed; retry]\n"));
                f->uf_retry = 1;
                return (SFRC_FAIL);
        }

        if (!f->uf_vfs_ufsfxp->fx_current)
                f->uf_vfs_ufsfxp->fx_current = f;

        mutex_exit(f->uf_vfs_lockp);

        f->uf_lf.lf_flags = 0;
        f->uf_lf.lf_lock  = LOCKFS_ELOCK;
        f->uf_retry = -ufsfx_tune.uft_fixstart_period;
        TRIVIA(("] "));
        return (SFRC_SUCCESS);
}

static sfrc_t
sf_found_trylck(ufs_failure_t *f)
{
        struct lockfs lockfs_status;

        TRIVIA(("[sf_found_trylck"));

        if (trylock_time_exceeded(f) > 0) {
                (void) set_state(f, UF_PANIC);
                TRIVIA((": failed] "));
                return (SFRC_FAIL);
        }

        if (!get_lockfs_status(f, &lockfs_status)) {
                (void) set_state(f, UF_PANIC);
                TRIVIA((": failed] "));
                return (SFRC_FAIL);
        }

        if (f->uf_lf_err == NO_ERROR)
                f->uf_lf.lf_key = lockfs_status.lf_key;

        if (!set_lockfs(f, &lockfs_status)) {
                (void) set_state(f, UF_PANIC);
                TRIVIA((": failed] "));
                return (SFRC_FAIL);
        }
        TRIVIA(("] "));
        return (SFRC_SUCCESS);
}

static sfrc_t
sf_set_locked(ufs_failure_t *f)
{
        TRIVIA(("[sf_set_locked"));

        f->uf_retry = -ufsfx_tune.uft_fixstart_period;

#if defined(DEBUG)
        if (f->uf_flags & UFSFX_REPAIR_START)
                TRIVIA(("clearing UFSFX_REPAIR_START "));
#endif /* DEBUG */

        f->uf_flags &= ~UFSFX_REPAIR_START;

        if (f->uf_s & UF_TRYLCK) {
                cmn_err(CE_WARN, "Error-locked %s: \"%s\"",
                    fs_name(f), f->uf_panic_str);

                if (f->uf_flags & UFSFX_LCKONLY)
                        cmn_err(CE_WARN, "Manual repair of %s required",
                            fs_name(f));
        }

        /*
         * just reset to current state
         */
#if defined(DEBUG)
        TRIVIA(("locked->locked "));
#endif /* DEBUG */

        TRIVIA(("] "));
        return (SFRC_SUCCESS);
}

static sfrc_t
sf_found_lock_fix_cmn(ufs_failure_t *f, ufs_failure_states_t s)
{
        time_t          toolong;
        extern time_t   time;
        struct buf      *bp                     = NULL;
        struct fs       *dfs;
        time_t           concerned, anxious;
        sfrc_t           rc                     = SFRC_FAIL;
        ulong_t          gb_size;

        TRIVIA(("[sf_found_lock_fix_cmn (\"%s\")", state_name(s)));

        if (s & UF_LOCKED) {
                ASSERT(MUTEX_HELD(&f->uf_mutex));

                toolong =
                    time > (ufsfx_tune.uft_too_long + f->uf_entered_tm);
                TRIVIA(("%stoolong", !toolong? "not": ""));
                HIDEOUS((": time:%ld, too long:%ld, entered_tm:%ld ",
                    time, ufsfx_tune.uft_too_long, f->uf_entered_tm));

                if (f->uf_flags & UFSFX_LCKUMOUNT) {
                        if (set_state(f, UF_UMOUNT)) {
                                TRIVIA(("] "));
                                rc = SFRC_SUCCESS;
                        } else {
                                TRIVIA((": failed] "));
                                f->uf_retry = 1;
                        }
                        return (rc);
                }
                if (!toolong) {
                        rc = SFRC_SUCCESS;
                } else {
                        if (!(f->uf_flags & UFSFX_REPAIR_START)) {
                                cmn_err(CE_WARN, "%s repair of %s not started.",
                                    (f->uf_flags & UFSFX_LCKONLY) ?
                                    "Manual" : "Automatic", fs_name(f));

                                f->uf_retry = ufsfx_tune.uft_long_err_period;
                        } else {
                                f->uf_retry = ufsfx_tune.uft_long_err_period;
                                cmn_err(CE_WARN, "Repair of %s is not timely; "
                                    "operator attention is required.",
                                    fs_name(f));
                        }
                        TRIVIA(("] "));
                        return (rc);
                }
        }

#if defined(DEBUG)
        else {
                ASSERT(s & UF_FIXING);
        }
#endif /* DEBUG */

        /*
         * get on disk superblock; force it to really
         * come from the disk
         */
        (void) bfinval(f->uf_dev, 0);
        bp = UFS_BREAD(f->uf_ufsvfsp, f->uf_dev, SBLOCK, SBSIZE);
        if (bp) {
                bp->b_flags |= (B_STALE | B_AGE);
                dfs = bp->b_un.b_fs;
        }

        if (!bp || (bp->b_flags & B_ERROR) || ((dfs->fs_magic != FS_MAGIC) &&
            (dfs->fs_magic != MTB_UFS_MAGIC))) {
                TRIVIA((": UFS_BREAD(SBLOCK) failed]\n"));
                f->uf_retry = 1;
                goto out;
        }

        /* fsck started but we haven't noticed yet? */
        if (!(s & UF_FIXING) && dfs->fs_clean == FSFIX) {
                if (!set_state(f, UF_FIXING)) {
                        TRIVIA((": failed]\n"));
                        f->uf_retry = 1;
                        goto out;
                }
        }

        /* fsck started but didn't succeed? */
        if ((s & UF_FIXING) && ((dfs->fs_clean == FSBAD) || !fsck_active(f))) {
                TRIVIA((": fs_clean: %d", (int)dfs->fs_clean));
                (void) set_state(f, UF_LOCKED);
                cmn_err(CE_WARN, "%s: Manual repair is necessary.", fs_name(f));
                f->uf_retry = ufsfx_tune.uft_long_err_period;
                goto out;
        }

        gb_size = (dfs->fs_size * dfs->fs_bshift) / GB;
        toolong = (time_t)((gb_size == 0? 1: gb_size) * SecondsPerGig);

        /* fsck started but doesn't seem to be proceeding? */
        if ((s & UF_FIXING) && dfs->fs_clean == FSFIX) {
                if (time > f->uf_entered_tm + toolong) {

                        cmn_err(CE_WARN,
                            "Repair completion timeout exceeded on %s; "
                            "manual fsck may be required", fs_name(f));
                        f->uf_retry = ufsfx_tune.uft_long_err_period;
                }
        }

        concerned = f->uf_entered_tm + (toolong / 3);
        anxious = f->uf_entered_tm + ((2 * toolong) / 3);

        if (time > concerned)
                pester_msg(f, time > anxious? CE_WARN: CE_NOTE);

        TRIVIA(("] "));

out:
        if (bp)
                brelse(bp);

        return (rc);
}

static sfrc_t
sf_found_umount(ufs_failure_t *f)
{
        extern time_t    time;
        sfrc_t           rc                     = SFRC_FAIL;
        struct vfs      *vfsp                   = f->uf_vfsp;
        struct ufsvfs   *ufsvfsp                = f->uf_ufsvfsp;
        int              toolong                = 0;
        int              err                    = 0;

        TRIVIA(("[sf_found_umount"));

        toolong = time > ufsfx_tune.uft_too_long + f->uf_entered_tm;
        if (toolong) {
                TRIVIA((": unmount time limit exceeded] "));
                goto out;
        }

        if (!vfsp || !ufsvfsp) {        /* trivial case */
                TRIVIA((": NULL vfsp and/or ufsvfsp, already unmounted?] "));
                goto out;
        }

        if (!ULOCKFS_IS_ELOCK(&ufsvfsp->vfs_ulockfs)) {
                TRIVIA((": !not error locked?"));
                err = EINVAL;
                goto out;
        }

        /* The vn_vfsunlock will be done in dounmount() [.../common/fs/vfs.c] */
        if (vn_vfswlock(vfsp->vfs_vnodecovered)) {
                TRIVIA((": couldn't lock coveredvp"));
                err = EBUSY;
                goto out;
        }

        if ((err = dounmount(vfsp, 0, kcred)) != 0) {

                /* take note, but not many alternatives here */
                mutex_enter(&uf_stats.ufst_mutex);
                uf_stats.ufst_unmount_failures++;
                mutex_exit(&uf_stats.ufst_mutex);

                TRIVIA((": unmount failed] "));
        } else {
                cmn_err(CE_NOTE, "unmounted error-locked %s", fs_name(f));
        }

out:
        if (toolong || (err != EBUSY && err != EAGAIN))
                rc = set_state(f, UF_NOTFIX);

        TRIVIA(("] "));
        return (rc);
}

static sfrc_t
sf_term_cmn(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
{
        extern time_t   time;
        sfrc_t          rc = SFRC_FAIL;

        TRIVIA(("[sf_term_cmn, action is %s, state is %s",
            act_name(a), state_name(s)));
        ASSERT(s & (UF_FIXED | UF_NOTFIX | UF_REPLICA));
        ASSERT(terminal_state(s));

        if (!f->uf_ufsvfsp && !(f->uf_s & (UF_UMOUNT | UF_NOTFIX))) {
                TRIVIA((": NULL ufsvfsp (state != UMOUNT | NOTFIX)]\n"));
                return (rc);
        }

        switch (a) {
        case UFA_SET:
                switch (s) {
                case UF_NOTFIX:
                case UF_FIXED:
                {
                        int need_lock_vfs;

                        if (f->uf_ufsvfsp && f->uf_vfs_lockp)
                                need_lock_vfs = !MUTEX_HELD(f->uf_vfs_lockp);
                        else
                                need_lock_vfs = 0;

                        if (need_lock_vfs && !mutex_tryenter(f->uf_vfs_lockp)) {
                                TRIVIA((": tryenter(vfslockp) fail; retry]\n"));
                                f->uf_retry = 1;
                                break;
                        }

                        f->uf_end_tm = time;
                        f->uf_lf.lf_lock = LOCKFS_OLOCK;
                        f->uf_retry = 0;

                        if (f->uf_vfs_ufsfxp)
                                f->uf_vfs_ufsfxp->fx_current = NULL;

                        if (need_lock_vfs)
                                mutex_exit(f->uf_vfs_lockp);

                        cmn_err(CE_NOTE, (s & UF_NOTFIX)? "Could not fix %s":
                            "%s is now accessible", fs_name(f));

                        if (s & UF_FIXED) {
                                mutex_enter(&uf_stats.ufst_mutex);
                                uf_stats.ufst_num_fixed++;
                                mutex_exit(&uf_stats.ufst_mutex);
                        }
                        (void) timeout(ufsfx_kill_fix_failure_thread,
                            (void *)(ufsfx_tune.uft_short_err_period * hz),
                            ufsfx_tune.uft_short_err_period * hz);
                        rc = SFRC_SUCCESS;
                        break;
                }
                case UF_REPLICA:

                        ASSERT(MUTEX_HELD(f->uf_vfs_lockp));

                        /* not actually a replica? */
                        if (f->uf_vfs_ufsfxp && f->uf_vfs_ufsfxp->fx_current &&
                            f->uf_vfs_ufsfxp->fx_current != f &&
                            !terminal_state(
                            f->uf_vfs_ufsfxp->fx_current->uf_s)) {

                                f->uf_orig = f->uf_vfs_ufsfxp->fx_current;
                                f->uf_retry = 0;
                                rc = SFRC_SUCCESS;
                        } else {
                                TRIVIA((": NULL fx_current]\n"));
                                f->uf_retry = 1;
                        }

                        break;

                default:
                        rc = set_state(f, UF_PANIC);
                        TRIVIA((": failed] "));
                        break;
                }
                break;

        case UFA_FOUND:
                /*
                 * XXX de-allocate these after some period?
                 * XXX or move to an historical list?
                 * XXX or have an ioctl which reaps them?
                 */
                /*
                 * For now, since we don't expect lots of failures
                 * to occur (to the point of memory shortages),
                 * just punt
                 */

                /* be sure we're not wasting cpu on old failures */
                if (f->uf_retry != 0) {
                        mutex_enter(&uf_stats.ufst_mutex);
                        uf_stats.ufst_cpu_waste++;
                        mutex_exit(&uf_stats.ufst_mutex);
                        f->uf_retry = 0;
                }
                rc = SFRC_SUCCESS;
                break;

        default:
                (void) set_state(f, UF_PANIC);
                TRIVIA((": failed] "));
                break;
        }

        TRIVIA(("] "));
        return (rc);
}

static sfrc_t
sf_panic(
        ufs_failure_t   *f,
        ufsa_t   a,
        ufs_failure_states_t     s)
{
        sfrc_t  rc = SFRC_FAIL;

        TRIVIA(("[sf_panic, action is %s, prev. state is %s",
            act_name(a), state_name(f->uf_s)));
        ASSERT(s & UF_PANIC);

        switch (a) {
        case UFA_SET:
                f->uf_retry = -ufsfx_tune.uft_short_err_period;
                rc = SFRC_SUCCESS;
                break;

        case UFA_FOUND:
        default:
                real_panic(f, " ");

                /* LINTED: warning: logical expression always true: op "||" */
                ASSERT(DEBUG);

                (void) set_state(f, UF_UMOUNT); /* XXX UF_NOTFIX? */

                break;
        }

        TRIVIA(("] "));
        return (rc);
}

/*
 * minimum state function
 */
static sfrc_t
sf_minimum(
        ufs_failure_t   *f,
        ufsa_t   a, /* LINTED argument unused in function: ignored */
        ufs_failure_states_t     ignored)
{
        sfrc_t rc = SFRC_FAIL;

        TRIVIA(("[sf_minimum, action is %s", act_name(a)));

        switch (a) {
        case UFA_SET:
                f->uf_retry = 0;
                /* FALLTHROUGH */

        case UFA_FOUND:
                rc = SFRC_SUCCESS;
                break;

        default:
                (void) set_state(f, UF_PANIC);
                TRIVIA((": failed] "));
                break;
        }

        TRIVIA(("] "));
        return (rc);
}

static int
state_trans_valid(ufs_failure_states_t from, ufs_failure_states_t to)
{
        ufsd_t  *s;
        int      valid;

        HIDEOUS(("[state_trans_valid"));

        if (from & to)
                return (1);

        s = get_state_desc(to);

        /*
         * extra test is necessary since we want UF_UNDEF = 0,
         * (to detect freshly allocated memory)
         * but can't check for that value with a bit test
         */
        valid = (to & UF_INIT)? from == s->ud_prev: from & s->ud_prev;

        HIDEOUS((": %svalid] ", valid? "": "in"));
        return (valid);
}

static int
terminal_state(ufs_failure_states_t state)
{
        ufsd_t  *s;

        HIDEOUS(("[terminal_state"));

        s = get_state_desc(state);

        HIDEOUS((": %sterminal] ", s->ud_attr.terminal? "": "not "));
        return ((int)s->ud_attr.terminal);
}

static void
alloc_lockfs_comment(ufs_failure_t *f, struct lockfs *lfp)
{
        MINUTE(("[alloc_lockfs_comment"));
        ASSERT(MUTEX_HELD(&f->uf_mutex));

        /*
         * ufs_fiolfs expects a kmem_alloc'ed comment;
         * it frees the comment if the lock fails
         * or else when the lock is unlocked.
         */

        f->uf_lf.lf_comment = kmem_zalloc(LOCKFS_MAXCOMMENTLEN, KM_NOSLEEP);
        if (f->uf_lf.lf_comment) {
                char    *from;
                size_t   len;

                /*
                 * use panic string if there's no previous comment
                 * or if we're setting the error lock
                 */
                if ((LOCKFS_IS_ELOCK(&f->uf_lf) || !lfp->lf_comment ||
                    lfp->lf_comlen <= 0)) {
                        from = f->uf_panic_str;
                        len = LOCKFS_MAXCOMMENTLEN;
                } else {
                        from = lfp->lf_comment;
                        len = lfp->lf_comlen;
                }

                bcopy(from, f->uf_lf.lf_comment, len);
                f->uf_lf.lf_comlen = len;

        } else {
                f->uf_lf.lf_comlen = 0;
        }
        MINUTE(("] "));
}

static int
set_lockfs(ufs_failure_t *f, struct lockfs *lfp)
{
        int     (*handle_lockfs_rc)(ufs_failure_t *);
        int       rc;

        MINUTE(("[set_lockfs"));
        ASSERT(MUTEX_HELD(&f->uf_mutex));
        ASSERT(!vfs_lock_held(f->uf_vfsp));
        ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));

        if (!f->uf_ufsvfsp) {
                MINUTE((": ufsvfsp is NULL]\n"));
                return (0);
        }

        ASSERT(MUTEX_NOT_HELD(&f->uf_ufsvfsp->vfs_ulockfs.ul_lock));

        if (!f->uf_ufsvfsp->vfs_root) {
                MINUTE((": vfs_root is NULL]\n"));
                return (0);
        }

        alloc_lockfs_comment(f, lfp);
        f->uf_lf_err = 0;

        if (!LOCKFS_IS_ELOCK(lfp)) {
                lfp->lf_lock = f->uf_lf.lf_lock = LOCKFS_ELOCK;
                VN_HOLD(f->uf_ufsvfsp->vfs_root);
                f->uf_lf_err =
                    ufs__fiolfs(f->uf_ufsvfsp->vfs_root,
                    &f->uf_lf, /* from_user */ 0, /* from_log */ 0);
                VN_RELE(f->uf_ufsvfsp->vfs_root);
        }

        handle_lockfs_rc = f->uf_lf_err != 0? lockfs_failure: lockfs_success;
        rc = handle_lockfs_rc(f);

        MINUTE(("] "));
        return (rc);
}

static int
lockfs_failure(ufs_failure_t *f)
{
        int     error;
        ufs_failure_states_t    s;

        TRIVIA(("[lockfs_failure"));
        ASSERT(MUTEX_HELD(&f->uf_mutex));

        if (!f->uf_ufsvfsp) {
                TRIVIA((": ufsvfsp is NULL]\n"));
                return (0);
        }

        error = f->uf_lf_err;
        switch (error) {
                        /* non-transient errors: */
        case EACCES:    /* disk/in-core metadata reconciliation failed  */
        case EPERM:     /* inode reconciliation failed; incore inode changed? */
        case EIO:       /* device is hard-locked or not responding */
        case EROFS:     /* device is write-locked */
        case EDEADLK:   /* can't lockfs; deadlock would result; */
                        /* Swapping or saving accounting records */
                        /* onto this fs can cause this errno. */

                MINOR(("ufs_fiolfs(\"%s\") of %s failed: %s (%d)",
                    fs_name(f), lock_name(&f->uf_lf),
                    err_name(error), error));

                /*
                 * if can't get lock, then fallback to panic, unless
                 * unless unmount was requested (although unmount will
                 * probably fail if the lock failed, so we'll panic
                 * anyway
                 */

                s = ((f->uf_flags & UFSFX_LCKUMOUNT) && error != EDEADLK) ?
                    UF_UMOUNT: UF_PANIC;

                if (!set_state(f, s)) {
                        real_panic(f, " ");
                        /*NOTREACHED*/
                        break;
                }
                break;


        case EBUSY:
        case EAGAIN:

                f->uf_retry = ufsfx_tune.uft_short_err_period;
                if (curthread->t_flag & T_DONTPEND) {
                        curthread->t_flag &= ~T_DONTPEND;

                } else if (!(f->uf_s & (UF_LOCKED | UF_FIXING))) {
                        ufs_failure_states_t state;
                        /*
                         * if we didn't know that the fix had started,
                         * take note
                         */
                        state = error == EBUSY? UF_LOCKED: UF_FIXING;
                        if (!set_state(f, state)) {
                                TRIVIA((": failed] "));
                                return (0);
                        }
                }
                break;

        default:        /* some other non-fatal error */
                MINOR(("lockfs(\"%s\") of %s returned %s (%d)",
                    lock_name(&f->uf_lf), fs_name(f),
                    err_name(f->uf_lf_err), f->uf_lf_err));

                f->uf_retry = ufsfx_tune.uft_short_err_period;
                break;

        case EINVAL:    /* unmounted? */
                (void) set_state(f, UF_NOTFIX);
                break;
        }
        TRIVIA(("] "));
        return (1);
}

static int
lockfs_success(ufs_failure_t *f)
{
        TRIVIA(("[lockfs_success"));
        ASSERT(MUTEX_HELD(&f->uf_mutex));

        if (!f->uf_ufsvfsp) {
                TRIVIA((": ufsvfsp is NULL]\n"));
                return (0);
        }

        switch (f->uf_lf.lf_lock) {
        case LOCKFS_ELOCK:      /* error lock worked */

                if (!set_state(f, UF_LOCKED)) {
                        TRIVIA((": failed] "));
                        return (0);
                }
                break;

        case LOCKFS_ULOCK:      /* unlock worked */
                /*
                 * how'd we get here?
                 * This should be done from fsck's unlock,
                 * not from this thread's context.
                 */
                cmn_err(CE_WARN, "Unlocked error-lock of %s", fs_name(f));
                ufsfx_unlockfs(f->uf_ufsvfsp);
                break;

        default:
                if (!set_state(f, UF_NOTFIX)) {
                        TRIVIA((": failed] "));
                        return (0);
                }
                break;
        }
        TRIVIA(("] "));
        return (1);
}

/*
 * when fsck is running it puts its pid into the lockfs
 * comment structure, prefaced by PIDSTR
 */
const char *PIDSTR = "[pid:";
static int
fsck_active(ufs_failure_t *f)
{
        char            *cp;
        int              i, found, errlocked;
        size_t           comlen;
        const int        PIDSTRLEN = (int)strlen(PIDSTR);
        struct ulockfs  *ulp = &f->uf_ufsvfsp->vfs_ulockfs;

        TRIVIA(("[fsck_active"));

        ASSERT(f);
        ASSERT(f->uf_s & UF_FIXING);
        ASSERT(MUTEX_HELD(&f->uf_mutex));
        ASSERT(f->uf_ufsvfsp);
        ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
        ASSERT(MUTEX_NOT_HELD(&ulp->ul_lock));

        mutex_enter(&ulp->ul_lock);
        cp = ulp->ul_lockfs.lf_comment;
        comlen = ulp->ul_lockfs.lf_comlen;
        errlocked = (int)ULOCKFS_IS_ELOCK(ulp);
        mutex_exit(&ulp->ul_lock);

        if (!cp || comlen == 0) {
                TRIVIA((": null comment or comlen <= 0, found:0]"));
                return (0);
        }

        for (found = i = 0; !found && i < (comlen - PIDSTRLEN); i++, cp++)
                found = strncmp(cp, PIDSTR, PIDSTRLEN) == 0;

        TRIVIA(("found:%d, is_elock:%d]", found, errlocked));
        return (errlocked & found);
}

static const char unknown_fs[]          = "<unknown fs>";
static const char null_failure[] = "<NULL ufs failure record; unknown fs>";
static const char mutated_vfs_bufp[]    = "<mutated vfs_bufp, unknown fs>";
static const char mutated_vfs_fs[]      = "<mutated vfs_fs, unknown fs>";

static char *
fs_name(ufs_failure_t *f)
{
        HIDEOUS(("[fs_name"));
        ASSERT(MUTEX_HELD(&f->uf_mutex));

        if (!f) {
                HIDEOUS((": failure ptr is NULL]\n"));
                return ((char *)null_failure);
        }

        if (f->uf_fsname[0] != '\0') {
                HIDEOUS((": return (uf_fsname)]\n"));
                return (f->uf_fsname);
        }

        if (MUTEX_HELD(f->uf_vfs_lockp)) {
                if (f->uf_bp != f->uf_ufsvfsp->vfs_bufp) {
                        HIDEOUS((": vfs_bufp mutated from 0x%p to 0x%p\n",
                            (void *)f->uf_bp, (void *)f->uf_ufsvfsp->vfs_bufp));
                        return ((char *)mutated_vfs_bufp);
                }
                if (f->uf_fs != f->uf_ufsvfsp->vfs_fs) {
                        HIDEOUS((": vfs_bufp mutated from 0x%p to 0x%p\n",
                            (void *)f->uf_fs, (void *)f->uf_ufsvfsp->vfs_fs));
                        return ((char *)mutated_vfs_fs);
                }
                if (f->uf_ufsvfsp && f->uf_bp && f->uf_fs &&
                    *f->uf_fs->fs_fsmnt != '\0') {
                        HIDEOUS((": return (fs_fsmnt)]\n"));
                        return (f->uf_fs->fs_fsmnt);
                }
        }

        HIDEOUS((": unknown file system]\n"));
        return ((char *)unknown_fs);
}

#if defined(DEBUG)
static char *
lock_name(struct lockfs *lfp)
{
        struct lock_description *l;
        char                    *lname;

        HIDEOUS(("[lock_name"));

        lname = lock_desc[0].ld_name;
        for (l = &lock_desc[1]; l->ld_name != NULL; l++) {
                if (lfp && lfp->lf_lock == l->ld_type) {
                        lname = l->ld_name;
                        break;
                }
        }
        HIDEOUS(("]"));
        return (lname);
}

static char *
state_name(ufs_failure_states_t state)
{
        ufsd_t  *s;

        HIDEOUS(("[state_name"));

        s = get_state_desc(state);

        HIDEOUS(("]"));
        return (s->ud_name);
}

static char *
err_name(int error)
{
        struct error_description *e;

        HIDEOUS(("[err_name"));

        for (e = &err_desc[1]; e->ed_name != NULL; e++) {
                if (error == e->ed_errno) {
                        HIDEOUS(("]"));
                        return (e->ed_name);
                }
        }
        HIDEOUS(("]"));
        return (err_desc[0].ed_name);
}

static char *
act_name(ufsa_t action)
{
        struct action_description *a;

        HIDEOUS(("[act_name"));

        for (a = &act_desc[1]; a->ad_name != NULL; a++) {
                if (action == a->ad_v) {
                        HIDEOUS(("]"));
                        return (a->ad_name);
                }
        }
        HIDEOUS(("]"));
        return (act_desc[0].ad_name);
}

/*
 * dump failure list
 */
static void
dump_uf_list(char *msg)
{
        ufs_failure_t   *f;
        int              i;
        int              list_was_locked = MUTEX_HELD(&ufs_fix.uq_mutex);

        if (!list_was_locked && !mutex_tryenter(&ufs_fix.uq_mutex)) {
                printf("dump_uf_list: couldn't get list lock\n");
                return;
        }

        if (msg) {
                printf("\n%s", msg);
        }
        printf("\ndump_uf_list:\n\tuq_lowat: %d, uq_ne: %d\n",
            ufs_fix.uq_lowat, ufs_fix.uq_ne);

        mutex_enter(&uf_stats.ufst_mutex);
        printf("\tuf_stats.current_races: %ld\n", uf_stats.ufst_current_races);
        printf("\tuf_stats.num_failed: %ld\n", uf_stats.ufst_num_failed);
        printf("\tuf_stats.num_fixed: %ld\n", uf_stats.ufst_num_fixed);
        printf("\tuf_stats.cpu_waste: %ld\n", uf_stats.ufst_cpu_waste);
        printf("\tuf_stats.lock_violations: %ld, unmount_failures: %ld\n",
            uf_stats.ufst_lock_violations, uf_stats.ufst_unmount_failures);
        mutex_exit(&uf_stats.ufst_mutex);

        for (f = ufs_fix.uq_ufhead, i = 1; f; f = f->uf_next, i++) {

                if (!mutex_tryenter(&f->uf_mutex)) {
                        printf("%d.\t\"skipped - try enter failed\"\n", i);
                        continue;
                }

                dump_uf(f, i);

                mutex_exit(&f->uf_mutex);
        }

        printf("\n");

        if (!list_was_locked)
                mutex_exit(&ufs_fix.uq_mutex);
}

static void
dump_uf(ufs_failure_t *f, int i)
{
        if (!f) {
                printf("dump_uf: NULL failure record\n");
                return;
        }

        printf("%d.\t\"%s\" is %s.\n",
            i, fs_name(f), state_name(f->uf_s));
        printf("\t\"%s\"\tAddr: 0x%p\n", f->uf_panic_str, (void *)f);
        printf("\tNext: 0x%p\t\tPrev: 0x%p\n",
            (void *)f->uf_next, (void *)f->uf_prev);

        if (f->uf_orig)
                printf("\tOriginal failure: 0x%p \"%s\"\n",
                    (void *)f->uf_orig, f->uf_orig->uf_panic_str);

        printf("\tUfsvfs: 0x%p\t\tVfs_lockp: 0x%p\n",
            (void *)f->uf_ufsvfsp, (void *)f->uf_vfs_lockp);
        printf("\tVfs_fsfxp: 0x%p\n", (void *)f->uf_vfs_ufsfxp);
        printf("\tVfs_bufp: 0x%p", (void *)f->uf_bp);

        if (f->uf_bp)
                printf("\t\tVfs_fs: 0x%p\n", (void *)f->uf_fs);
        else
                printf("\n");

        printf("\tBegin: 0x%lx\tEntered: 0x%lx\tEnd: 0x%lx\n",
            f->uf_begin_tm, f->uf_entered_tm, f->uf_end_tm);

        printf("\tFlags: (%d) %s%s%s%s", f->uf_flags,
            f->uf_flags & UFSFX_LCKONLY?         "\"lock only\" "       : "",
            f->uf_flags & UFSFX_LCKUMOUNT?       "\"lock+unmount\" "    : "",
            f->uf_flags & UFSFX_REPAIR_START? "\"started repair\" "     : "",
            f->uf_flags == 0?                "<none>"               : "");

        printf("\tRetry: %ld seconds\n", f->uf_retry);

        printf("\tLockfs:\ttype: %s\terror: %s (%d)\n",
            lock_name(&f->uf_lf), err_name(f->uf_lf_err), f->uf_lf_err);

}
#endif /* DEBUG */

/*
 * returns # of ufs_failures in a non-terminal state on queue
 * used to coordinate with hlock thread (see ufs_thread.c)
 * and to determine when the error lock thread may exit
 */

int
ufsfx_get_failure_qlen(void)
{
        ufs_failure_t   *f;
        ufsd_t          *s;
        int              qlen = 0;

        MINUTE(("[ufsfx_get_failure_qlen"));

        if (!mutex_tryenter(&ufs_fix.uq_mutex))
                return (-1);

        /*
         * walk down failure list
         */

        for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

                if (!mutex_tryenter(&f->uf_mutex))
                        continue;

                s = get_state_desc(f->uf_s);

                if (s->ud_attr.terminal) {
                        mutex_exit(&f->uf_mutex);
                        continue;
                }

                MINUTE((": found: %s, \"%s: %s\"\n",
                    fs_name(f), state_name(f->uf_s), f->uf_panic_str));

                qlen++;
                mutex_exit(&f->uf_mutex);
        }

        mutex_exit(&ufs_fix.uq_mutex);

        MINUTE((": qlen=%d]\n", qlen));

        return (qlen);
}

/*
 * timeout routine
 *  called to shutdown fix failure thread and server daemon
 */
static void
ufsfx_kill_fix_failure_thread(void *arg)
{
        clock_t odelta = (clock_t)arg;
        int     qlen;

        MAJOR(("[ufsfx_kill_fix_failure_thread"));

        qlen = ufsfx_get_failure_qlen();

        if (qlen < 0) {
                clock_t delta;

                delta = odelta << 1;
                if (delta <= 0)
                        delta = INT_MAX;

                (void) timeout(ufsfx_kill_fix_failure_thread,
                    (void *)delta, delta);
                MAJOR((": rescheduled"));

        } else if (qlen == 0) {
                ufs_thread_exit(&ufs_fix);
                MAJOR((": killed"));
        }
        /*
         * else
         *  let timeout expire
         */
        MAJOR(("]\n"));
}