/*      $NetBSD: sl811hs.c,v 1.112 2022/05/03 20:52:32 andvar Exp $     */

/*
 * Not (c) 2007 Matthew Orgass
 * This file is public domain, meaning anyone can make any use of part or all
 * of this file including copying into other works without credit.  Any use,
 * modified or not, is solely the responsibility of the user.  If this file is
 * part of a collection then use in the collection is governed by the terms of
 * the collection.
 */

/*
 * Cypress/ScanLogic SL811HS/T USB Host Controller
 * Datasheet, Errata, and App Note available at www.cypress.com
 *
 * Uses: Ratoc CFU1U PCMCIA USB Host Controller, Nereid X68k USB HC, ISA
 * HCs.  The Ratoc CFU2 uses a different chip.
 *
 * This chip puts the serial in USB.  It implements USB by means of an eight
 * bit I/O interface.  It can be used for ISA, PCMCIA/CF, parallel port,
 * serial port, or any eight bit interface.  It has 256 bytes of memory, the
 * first 16 of which are used for register access.  There are two sets of
 * registers for sending individual bus transactions.  Because USB is polled,
 * this organization means that some amount of card access must often be made
 * when devices are attached, even if when they are not directly being used.
 * A per-ms frame interrupt is necessary and many devices will poll with a
 * per-frame bulk transfer.
 *
 * It is possible to write a little over two bytes to the chip (auto
 * incremented) per full speed byte time on the USB.  Unfortunately,
 * auto-increment does not work reliably so write and bus speed is
 * approximately the same for full speed devices.
 *
 * In addition to the 240 byte packet size limit for isochronous transfers,
 * this chip has no means of determining the current frame number other than
 * getting all 1ms SOF interrupts, which is not always possible even on a fast
 * system.  Isochronous transfers guarantee that transfers will never be
 * retried in a later frame, so this can cause problems with devices beyond
 * the difficulty in actually performing the transfer most frames.  I tried
 * implementing isoc transfers and was able to play CD-derrived audio via an
 * iMic on a 2GHz PC, however it would still be interrupted at times and
 * once interrupted, would stay out of sync.  All isoc support has been
 * removed.
 *
 * BUGS: all chip revisions have problems with low speed devices through hubs.
 * The chip stops generating SOF with hubs that send SE0 during SOF.  See
 * comment in dointr().  All performance enhancing features of this chip seem
 * not to work properly, most confirmed buggy in errata doc.
 *
 */

/*
 * The hard interrupt is the main entry point.  Start, callbacks, and repeat
 * are the only others called frequently.
 *
 * Since this driver attaches to pcmcia, card removal at any point should be
 * expected and not cause panics or infinite loops.
 */

/*
 * XXX TODO:
 *   copy next output packet while transferring
 *   usb suspend
 *   could keep track of known values of all buffer space?
 *   combined print/log function for errors
 *
 *   ub_usepolling support is untested and may not work
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: sl811hs.c,v 1.112 2022/05/03 20:52:32 andvar Exp $");

#ifdef _KERNEL_OPT
#include "opt_slhci.h"
#include "opt_usb.h"
#endif

#include <sys/param.h>

#include <sys/bus.h>
#include <sys/cpu.h>
#include <sys/device.h>
#include <sys/gcq.h>
#include <sys/intr.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

#include <dev/usb/usb.h>
#include <dev/usb/usbdi.h>
#include <dev/usb/usbdivar.h>
#include <dev/usb/usbhist.h>
#include <dev/usb/usb_mem.h>
#include <dev/usb/usbdevs.h>
#include <dev/usb/usbroothub.h>

#include <dev/ic/sl811hsreg.h>
#include <dev/ic/sl811hsvar.h>

#define Q_CB 0                          /* Control/Bulk */
#define Q_NEXT_CB 1
#define Q_MAX_XFER Q_CB
#define Q_CALLBACKS 2
#define Q_MAX Q_CALLBACKS

#define F_AREADY                (0x00000001)
#define F_BREADY                (0x00000002)
#define F_AINPROG               (0x00000004)
#define F_BINPROG               (0x00000008)
#define F_LOWSPEED              (0x00000010)
#define F_UDISABLED             (0x00000020) /* Consider disabled for USB */
#define F_NODEV                 (0x00000040)
#define F_ROOTINTR              (0x00000080)
#define F_REALPOWER             (0x00000100) /* Actual power state */
#define F_POWER                 (0x00000200) /* USB reported power state */
#define F_ACTIVE                (0x00000400)
#define F_CALLBACK              (0x00000800) /* Callback scheduled */
#define F_SOFCHECK1             (0x00001000)
#define F_SOFCHECK2             (0x00002000)
#define F_CRESET                (0x00004000) /* Reset done not reported */
#define F_CCONNECT              (0x00008000) /* Connect change not reported */
#define F_RESET                 (0x00010000)
#define F_ISOC_WARNED           (0x00020000)
#define F_LSVH_WARNED           (0x00040000)

#define F_DISABLED              (F_NODEV|F_UDISABLED)
#define F_CHANGE                (F_CRESET|F_CCONNECT)

#ifdef SLHCI_TRY_LSVH
unsigned int slhci_try_lsvh = 1;
#else
unsigned int slhci_try_lsvh = 0;
#endif

#define ADR 0
#define LEN 1
#define PID 2
#define DEV 3
#define STAT 2
#define CONT 3

#define A 0
#define B 1

static const uint8_t slhci_tregs[2][4] =
{{SL11_E0ADDR, SL11_E0LEN, SL11_E0PID, SL11_E0DEV },
 {SL11_E1ADDR, SL11_E1LEN, SL11_E1PID, SL11_E1DEV }};

#define PT_ROOT_CTRL    0
#define PT_ROOT_INTR    1
#define PT_CTRL_SETUP   2
#define PT_CTRL_DATA    3
#define PT_CTRL_STATUS  4
#define PT_INTR         5
#define PT_BULK         6
#define PT_MAX          6

#ifdef SLHCI_DEBUG
#define SLHCI_MEM_ACCOUNTING
#endif

/*
 * Maximum allowable reserved bus time.  Since intr/isoc transfers have
 * unconditional priority, this is all that ensures control and bulk transfers
 * get a chance.  It is a single value for all frames since all transfers can
 * use multiple consecutive frames if an error is encountered.  Note that it
 * is not really possible to fill the bus with transfers, so this value should
 * be on the low side.  Defaults to giving a warning unless SLHCI_NO_OVERTIME
 * is defined.  Full time is 12000 - END_BUSTIME.
 */
#ifndef SLHCI_RESERVED_BUSTIME
#define SLHCI_RESERVED_BUSTIME 5000
#endif

/*
 * Rate for "exceeds reserved bus time" warnings (default) or errors.
 * Warnings only happen when an endpoint open causes the time to go above
 * SLHCI_RESERVED_BUSTIME, not if it is already above.
 */
#ifndef SLHCI_OVERTIME_WARNING_RATE
#define SLHCI_OVERTIME_WARNING_RATE { 60, 0 } /* 60 seconds */
#endif
static const struct timeval reserved_warn_rate = SLHCI_OVERTIME_WARNING_RATE;

/*
 * For EOF, the spec says 42 bit times, plus (I think) a possible hub skew of
 * 20 bit times.  By default leave 66 bit times to start the transfer beyond
 * the required time.  Units are full-speed bit times (a bit over 5us per 64).
 * Only multiples of 64 are significant.
 */
#define SLHCI_STANDARD_END_BUSTIME 128
#ifndef SLHCI_EXTRA_END_BUSTIME
#define SLHCI_EXTRA_END_BUSTIME 0
#endif

#define SLHCI_END_BUSTIME (SLHCI_STANDARD_END_BUSTIME+SLHCI_EXTRA_END_BUSTIME)

/*
 * This is an approximation of the USB worst-case timings presented on p. 54 of
 * the USB 1.1 spec translated to full speed bit times.
 * FS = full speed with handshake, FSII = isoc in, FSIO = isoc out,
 * FSI = isoc (worst case), LS = low speed
 */
#define SLHCI_FS_CONST          114
#define SLHCI_FSII_CONST        92
#define SLHCI_FSIO_CONST        80
#define SLHCI_FSI_CONST         92
#define SLHCI_LS_CONST          804
#ifndef SLHCI_PRECICE_BUSTIME
/*
 * These values are < 3% too high (compared to the multiply and divide) for
 * max sized packets.
 */
#define SLHCI_FS_DATA_TIME(len) (((u_int)(len)<<3)+(len)+((len)>>1))
#define SLHCI_LS_DATA_TIME(len) (((u_int)(len)<<6)+((u_int)(len)<<4))
#else
#define SLHCI_FS_DATA_TIME(len) (56*(len)/6)
#define SLHCI_LS_DATA_TIME(len) (449*(len)/6)
#endif

/*
 * Set SLHCI_WAIT_SIZE to the desired maximum size of single FS transfer
 * to poll for after starting a transfer.  64 gets all full speed transfers.
 * Note that even if 0 polling will occur if data equal or greater than the
 * transfer size is copied to the chip while the transfer is in progress.
 * Setting SLHCI_WAIT_TIME to -12000 will disable polling.
 */
#ifndef SLHCI_WAIT_SIZE
#define SLHCI_WAIT_SIZE 8
#endif
#ifndef SLHCI_WAIT_TIME
#define SLHCI_WAIT_TIME (SLHCI_FS_CONST + \
    SLHCI_FS_DATA_TIME(SLHCI_WAIT_SIZE))
#endif
const int slhci_wait_time = SLHCI_WAIT_TIME;

#ifndef SLHCI_MAX_RETRIES
#define SLHCI_MAX_RETRIES 3
#endif

/* Check IER values for corruption after this many unrecognized interrupts. */
#ifndef SLHCI_IER_CHECK_FREQUENCY
#ifdef SLHCI_DEBUG
#define SLHCI_IER_CHECK_FREQUENCY 1
#else
#define SLHCI_IER_CHECK_FREQUENCY 100
#endif
#endif

/* Note that buffer points to the start of the buffer for this transfer.  */
struct slhci_pipe {
        struct usbd_pipe pipe;
        struct usbd_xfer *xfer;         /* xfer in progress */
        uint8_t         *buffer;        /* I/O buffer (if needed) */
        struct gcq      ap;             /* All pipes */
        struct gcq      to;             /* Timeout list */
        struct gcq      xq;             /* Xfer queues */
        unsigned int    pflags;         /* Pipe flags */
#define PF_GONE         (0x01)          /* Pipe is on disabled device */
#define PF_TOGGLE       (0x02)          /* Data toggle status */
#define PF_LS           (0x04)          /* Pipe is low speed */
#define PF_PREAMBLE     (0x08)          /* Needs preamble */
        Frame           to_frame;       /* Frame number for timeout */
        Frame           frame;          /* Frame number for intr xfer */
        Frame           lastframe;      /* Previous frame number for intr */
        uint16_t        bustime;        /* Worst case bus time usage */
        uint16_t        newbustime[2];  /* new bustimes (see index below) */
        uint8_t         tregs[4];       /* ADR, LEN, PID, DEV */
        uint8_t         newlen[2];      /* 0 = short data, 1 = ctrl data */
        uint8_t         newpid;         /* for ctrl */
        uint8_t         wantshort;      /* last xfer must be short */
        uint8_t         control;        /* Host control register settings */
        uint8_t         nerrs;          /* Current number of errors */
        uint8_t         ptype;          /* Pipe type */
};

#define SLHCI_BUS2SC(bus)       ((bus)->ub_hcpriv)
#define SLHCI_PIPE2SC(pipe)     SLHCI_BUS2SC((pipe)->up_dev->ud_bus)
#define SLHCI_XFER2SC(xfer)     SLHCI_BUS2SC((xfer)->ux_bus)

#define SLHCI_PIPE2SPIPE(pipe)  ((struct slhci_pipe *)(pipe))
#define SLHCI_XFER2SPIPE(xfer)  SLHCI_PIPE2SPIPE((xfer)->ux_pipe)

#define SLHCI_XFER_TYPE(x)      (SLHCI_XFER2SPIPE(xfer)->ptype)

#ifdef SLHCI_PROFILE_TRANSFER
#if defined(__mips__)
/*
 * MIPS cycle counter does not directly count cpu cycles but is a different
 * fraction of cpu cycles depending on the cpu.
 */
typedef uint32_t cc_type;
#define CC_TYPE_FMT "%u"
#define slhci_cc_set(x) __asm volatile ("mfc0 %[cc], $9\n\tnop\n\tnop\n\tnop" \
    : [cc] "=r"(x))
#elif defined(__i386__)
typedef uint64_t cc_type;
#define CC_TYPE_FMT "%llu"
#define slhci_cc_set(x) __asm volatile ("rdtsc" : "=A"(x))
#else
#error "SLHCI_PROFILE_TRANSFER not implemented on this MACHINE_ARCH (see sys/dev/ic/sl811hs.c)"
#endif
struct slhci_cc_time {
        cc_type start;
        cc_type stop;
        unsigned int miscdata;
};
#ifndef SLHCI_N_TIMES
#define SLHCI_N_TIMES 200
#endif
struct slhci_cc_times {
        struct slhci_cc_time times[SLHCI_N_TIMES];
        int current;
        int wraparound;
};

static struct slhci_cc_times t_ab[2];
static struct slhci_cc_times t_abdone;
static struct slhci_cc_times t_copy_to_dev;
static struct slhci_cc_times t_copy_from_dev;
static struct slhci_cc_times t_intr;
static struct slhci_cc_times t_lock;
static struct slhci_cc_times t_delay;
static struct slhci_cc_times t_hard_int;
static struct slhci_cc_times t_callback;

static inline void
start_cc_time(struct slhci_cc_times *times, unsigned int misc) {
        times->times[times->current].miscdata = misc;
        slhci_cc_set(times->times[times->current].start);
}
static inline void
stop_cc_time(struct slhci_cc_times *times) {
        slhci_cc_set(times->times[times->current].stop);
        if (++times->current >= SLHCI_N_TIMES) {
                times->current = 0;
                times->wraparound = 1;
        }
}

void slhci_dump_cc_times(int);

void
slhci_dump_cc_times(int n) {
        struct slhci_cc_times *times;
        int i;

        switch (n) {
        default:
        case 0:
                printf("USBA start transfer to intr:\n");
                times = &t_ab[A];
                break;
        case 1:
                printf("USBB start transfer to intr:\n");
                times = &t_ab[B];
                break;
        case 2:
                printf("abdone:\n");
                times = &t_abdone;
                break;
        case 3:
                printf("copy to device:\n");
                times = &t_copy_to_dev;
                break;
        case 4:
                printf("copy from device:\n");
                times = &t_copy_from_dev;
                break;
        case 5:
                printf("intr to intr:\n");
                times = &t_intr;
                break;
        case 6:
                printf("lock to release:\n");
                times = &t_lock;
                break;
        case 7:
                printf("delay time:\n");
                times = &t_delay;
                break;
        case 8:
                printf("hard interrupt enter to exit:\n");
                times = &t_hard_int;
                break;
        case 9:
                printf("callback:\n");
                times = &t_callback;
                break;
        }

        if (times->wraparound)
                for (i = times->current + 1; i < SLHCI_N_TIMES; i++)
                        printf("start " CC_TYPE_FMT " stop " CC_TYPE_FMT
                            " difference %8i miscdata %#x\n",
                            times->times[i].start, times->times[i].stop,
                            (int)(times->times[i].stop -
                            times->times[i].start), times->times[i].miscdata);

        for (i = 0; i < times->current; i++)
                printf("start " CC_TYPE_FMT " stop " CC_TYPE_FMT
                    " difference %8i miscdata %#x\n", times->times[i].start,
                    times->times[i].stop, (int)(times->times[i].stop -
                    times->times[i].start), times->times[i].miscdata);
}
#else
#define start_cc_time(x, y)
#define stop_cc_time(x)
#endif /* SLHCI_PROFILE_TRANSFER */

typedef usbd_status (*LockCallFunc)(struct slhci_softc *, struct slhci_pipe
    *, struct usbd_xfer *);

struct usbd_xfer * slhci_allocx(struct usbd_bus *, unsigned int);
void slhci_freex(struct usbd_bus *, struct usbd_xfer *);
static void slhci_get_lock(struct usbd_bus *, kmutex_t **);

usbd_status slhci_transfer(struct usbd_xfer *);
usbd_status slhci_start(struct usbd_xfer *);
usbd_status slhci_root_start(struct usbd_xfer *);
usbd_status slhci_open(struct usbd_pipe *);

static int slhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *,
    void *, int);

/*
 * slhci_supported_rev, slhci_preinit, slhci_attach, slhci_detach,
 * slhci_activate
 */

void slhci_abort(struct usbd_xfer *);
void slhci_close(struct usbd_pipe *);
void slhci_clear_toggle(struct usbd_pipe *);
void slhci_poll(struct usbd_bus *);
void slhci_done(struct usbd_xfer *);
void slhci_void(void *);

/* lock entry functions */

#ifdef SLHCI_MEM_ACCOUNTING
void slhci_mem_use(struct usbd_bus *, int);
#endif

void slhci_reset_entry(void *);
usbd_status slhci_lock_call(struct slhci_softc *, LockCallFunc,
    struct slhci_pipe *, struct usbd_xfer *);
void slhci_start_entry(struct slhci_softc *, struct slhci_pipe *);
void slhci_callback_entry(void *arg);
void slhci_do_callback(struct slhci_softc *, struct usbd_xfer *);

/* slhci_intr */

void slhci_main(struct slhci_softc *);

/* in lock functions */

static void slhci_write(struct slhci_softc *, uint8_t, uint8_t);
static uint8_t slhci_read(struct slhci_softc *, uint8_t);
static void slhci_write_multi(struct slhci_softc *, uint8_t, uint8_t *, int);
static void slhci_read_multi(struct slhci_softc *, uint8_t, uint8_t *, int);

static void slhci_waitintr(struct slhci_softc *, int);
static int slhci_dointr(struct slhci_softc *);
static void slhci_abdone(struct slhci_softc *, int);
static void slhci_tstart(struct slhci_softc *);
static void slhci_dotransfer(struct slhci_softc *);

static void slhci_callback(struct slhci_softc *);
static void slhci_enter_xfer(struct slhci_softc *, struct slhci_pipe *);
static void slhci_enter_xfers(struct slhci_softc *);
static void slhci_queue_timed(struct slhci_softc *, struct slhci_pipe *);
static void slhci_xfer_timer(struct slhci_softc *, struct slhci_pipe *);

static void slhci_callback_schedule(struct slhci_softc *);
static void slhci_do_callback_schedule(struct slhci_softc *);
#if 0
void slhci_pollxfer(struct slhci_softc *, struct usbd_xfer *); /* XXX */
#endif

static usbd_status slhci_do_poll(struct slhci_softc *, struct slhci_pipe *,
    struct usbd_xfer *);
static usbd_status slhci_lsvh_warn(struct slhci_softc *, struct slhci_pipe *,
    struct usbd_xfer *);
static usbd_status slhci_isoc_warn(struct slhci_softc *, struct slhci_pipe *,
    struct usbd_xfer *);
static usbd_status slhci_open_pipe(struct slhci_softc *, struct slhci_pipe *,
    struct usbd_xfer *);
static usbd_status slhci_close_pipe(struct slhci_softc *, struct slhci_pipe *,
    struct usbd_xfer *);
static usbd_status slhci_do_abort(struct slhci_softc *, struct slhci_pipe *,
    struct usbd_xfer *);
static usbd_status slhci_halt(struct slhci_softc *, struct slhci_pipe *,
    struct usbd_xfer *);

static void slhci_intrchange(struct slhci_softc *, uint8_t);
static void slhci_drain(struct slhci_softc *);
static void slhci_reset(struct slhci_softc *);
static int slhci_reserve_bustime(struct slhci_softc *, struct slhci_pipe *,
    int);
static void slhci_insert(struct slhci_softc *);

static usbd_status slhci_clear_feature(struct slhci_softc *, unsigned int);
static usbd_status slhci_set_feature(struct slhci_softc *, unsigned int);
static void slhci_get_status(struct slhci_softc *, usb_port_status_t *);

#define SLHCIHIST_FUNC()        USBHIST_FUNC()
#define SLHCIHIST_CALLED()      USBHIST_CALLED(slhcidebug)

#ifdef SLHCI_DEBUG
static int slhci_memtest(struct slhci_softc *);

void slhci_log_buffer(struct usbd_xfer *);
void slhci_log_req(usb_device_request_t *);
void slhci_log_dumpreg(void);
void slhci_log_xfer(struct usbd_xfer *);
void slhci_log_spipe(struct slhci_pipe *);
void slhci_print_intr(void);
void slhci_log_sc(void);
void slhci_log_slreq(struct slhci_pipe *);

/* Constified so you can read the values from ddb */
const int SLHCI_D_TRACE =       0x0001;
const int SLHCI_D_MSG =         0x0002;
const int SLHCI_D_XFER =        0x0004;
const int SLHCI_D_MEM =         0x0008;
const int SLHCI_D_INTR =        0x0010;
const int SLHCI_D_SXFER =       0x0020;
const int SLHCI_D_ERR =         0x0080;
const int SLHCI_D_BUF =         0x0100;
const int SLHCI_D_SOFT =        0x0200;
const int SLHCI_D_WAIT =        0x0400;
const int SLHCI_D_ROOT =        0x0800;
/* SOF/NAK alone normally ignored, SOF also needs D_INTR */
const int SLHCI_D_SOF =         0x1000;
const int SLHCI_D_NAK =         0x2000;

int slhcidebug = 0x1cbc; /* 0xc8c; */ /* 0xffff; */ /* 0xd8c; */

SYSCTL_SETUP(sysctl_hw_slhci_setup, "sysctl hw.slhci setup")
{
        int err;
        const struct sysctlnode *rnode;
        const struct sysctlnode *cnode;

        err = sysctl_createv(clog, 0, NULL, &rnode,
            CTLFLAG_PERMANENT, CTLTYPE_NODE, "slhci",
            SYSCTL_DESCR("slhci global controls"),
            NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);

        if (err)
                goto fail;

        /* control debugging printfs */
        err = sysctl_createv(clog, 0, &rnode, &cnode,
            CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT,
            "debug", SYSCTL_DESCR("Enable debugging output"),
            NULL, 0, &slhcidebug, sizeof(slhcidebug), CTL_CREATE, CTL_EOL);
        if (err)
                goto fail;

        return;
fail:
        aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err);
}

struct slhci_softc *ssc;

#define SLHCI_DEXEC(x, y) do { if ((slhcidebug & SLHCI_ ## x)) { y; } \
} while (/*CONSTCOND*/ 0)
#define DDOLOG(f, a, b, c, d) do { KERNHIST_LOG(usbhist, f, a, b, c, d); \
} while (/*CONSTCOND*/0)
#define DLOG(x, f, a, b, c, d) SLHCI_DEXEC(x, DDOLOG(f, a, b, c, d))

/*
 * DDOLOGBUF logs a buffer up to 8 bytes at a time. No identifier so that we
 * can make it a real function.
 */
static void
DDOLOGBUF(uint8_t *buf, unsigned int length)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        int i;

        for(i = 0; i + 8 <= length; i += 8)
                DDOLOG("%.4x %.4x %.4x %.4x", (buf[i] << 8) | buf[i+1],
                    (buf[i+2] << 8) | buf[i+3], (buf[i+4] << 8) | buf[i+5],
                    (buf[i+6] << 8) | buf[i+7]);
        if (length == i + 7)
                DDOLOG("%.4x %.4x %.4x %.2x", (buf[i] << 8) | buf[i+1],
                    (buf[i+2] << 8) | buf[i+3], (buf[i+4] << 8) | buf[i+5],
                    buf[i+6]);
        else if (length == i + 6)
                DDOLOG("%.4x %.4x %.4x", (buf[i] << 8) | buf[i+1],
                    (buf[i+2] << 8) | buf[i+3], (buf[i+4] << 8) | buf[i+5], 0);
        else if (length == i + 5)
                DDOLOG("%.4x %.4x %.2x", (buf[i] << 8) | buf[i+1],
                    (buf[i+2] << 8) | buf[i+3], buf[i+4], 0);
        else if (length == i + 4)
                DDOLOG("%.4x %.4x", (buf[i] << 8) | buf[i+1],
                    (buf[i+2] << 8) | buf[i+3], 0,0);
        else if (length == i + 3)
                DDOLOG("%.4x %.2x", (buf[i] << 8) | buf[i+1], buf[i+2], 0,0);
        else if (length == i + 2)
                DDOLOG("%.4x", (buf[i] << 8) | buf[i+1], 0,0,0);
        else if (length == i + 1)
                DDOLOG("%.2x", buf[i], 0,0,0);
}
#define DLOGBUF(x, b, l) SLHCI_DEXEC(x, DDOLOGBUF(b, l))

#define DDOLOGCTRL(x)   do {                                            \
    DDOLOG("CTRL suspend=%jd", !!((x) & SL11_CTRL_SUSPEND), 0, 0, 0);   \
    DDOLOG("CTRL ls     =%jd  jk     =%jd  reset  =%jd  sof    =%jd",   \
        !!((x) & SL11_CTRL_LOWSPEED), !!((x) & SL11_CTRL_JKSTATE),      \
        !!((x) & SL11_CTRL_RESETENGINE), !!((x) & SL11_CTRL_ENABLESOF));\
} while (0)

#define DDOLOGISR(r)    do {                                            \
    DDOLOG("ISR  data   =%jd  det/res=%jd  insert =%jd  sof    =%jd",   \
        !!((r) & SL11_ISR_DATA), !!((r) & SL11_ISR_RESUME),             \
        !!((r) & SL11_ISR_INSERT), !!!!((r) & SL11_ISR_SOF));           \
    DDOLOG("ISR             babble =%jd  usbb   =%jd  usba   =%jd",     \
        !!((r) & SL11_ISR_BABBLE), !!((r) & SL11_ISR_USBB),             \
        !!((r) & SL11_ISR_USBA), 0);                                    \
} while (0)

#define DDOLOGIER(r)    do {                                            \
    DDOLOG("IER              det/res=%d  insert =%d  sof    =%d",       \
        !!((r) & SL11_IER_RESUME),                                      \
        !!((r) & SL11_IER_INSERT), !!!!((r) & SL11_IER_SOF), 0);                \
    DDOLOG("IER              babble =%d  usbb   =%d  usba   =%d",       \
        !!((r) & SL11_IER_BABBLE), !!((r) & SL11_IER_USBB),             \
        !!((r) & SL11_IER_USBA), 0);                                    \
} while (0)

#define DDOLOGSTATUS(s) do {                                            \
    DDOLOG("STAT stall   =%d  nak     =%d  overflow =%d  setup   =%d",  \
        !!((s) & SL11_EPSTAT_STALL), !!((s) & SL11_EPSTAT_NAK),         \
        !!((s) & SL11_EPSTAT_OVERFLOW), !!((s) & SL11_EPSTAT_SETUP));   \
    DDOLOG("STAT sequence=%d  timeout =%d  error    =%d  ack     =%d",  \
        !!((s) & SL11_EPSTAT_SEQUENCE), !!((s) & SL11_EPSTAT_TIMEOUT),  \
        !!((s) & SL11_EPSTAT_ERROR), !!((s) & SL11_EPSTAT_ACK));        \
} while (0)

#define DDOLOGEPCTRL(r) do {                                            \
    DDOLOG("CTRL preamble=%d  toggle  =%d  sof     =%d  iso     =%d",   \
        !!((r) & SL11_EPCTRL_PREAMBLE), !!((r) & SL11_EPCTRL_DATATOGGLE),\
        !!((r) & SL11_EPCTRL_SOF), !!((r) & SL11_EPCTRL_ISO));          \
    DDOLOG("CTRL              out     =%d  enable  =%d  arm     =%d",   \
        !!((r) & SL11_EPCTRL_DIRECTION),                                \
        !!((r) & SL11_EPCTRL_ENABLE), !!((r) & SL11_EPCTRL_ARM), 0);    \
} while (0)

#define DDOLOGEPSTAT(r) do {                                            \
    DDOLOG("STAT stall   =%d  nak     =%d  overflow =%d  setup   =%d",  \
        !!((r) & SL11_EPSTAT_STALL), !!((r) & SL11_EPSTAT_NAK),         \
        !!((r) & SL11_EPSTAT_OVERFLOW), !!((r) & SL11_EPSTAT_SETUP));   \
    DDOLOG("STAT sequence=%d  timeout =%d  error    =%d  ack   =%d",    \
        !!((r) & SL11_EPSTAT_SEQUENCE), !!((r) & SL11_EPSTAT_TIMEOUT),  \
        !!((r) & SL11_EPSTAT_ERROR), !!((r) & SL11_EPSTAT_ACK));        \
} while (0)
#else /* now !SLHCI_DEBUG */
#define slhcidebug 0
#define slhci_log_spipe(spipe) ((void)0)
#define slhci_log_xfer(xfer) ((void)0)
#define SLHCI_DEXEC(x, y) ((void)0)
#define DDOLOG(f, a, b, c, d) ((void)0)
#define DLOG(x, f, a, b, c, d) ((void)0)
#define DDOLOGBUF(b, l) ((void)0)
#define DLOGBUF(x, b, l) ((void)0)
#define DDOLOGCTRL(x) ((void)0)
#define DDOLOGISR(r) ((void)0)
#define DDOLOGIER(r) ((void)0)
#define DDOLOGSTATUS(s) ((void)0)
#define DDOLOGEPCTRL(r) ((void)0)
#define DDOLOGEPSTAT(r) ((void)0)
#endif /* SLHCI_DEBUG */

#ifdef DIAGNOSTIC
#define LK_SLASSERT(exp, sc, spipe, xfer, ext) do {                     \
        if (!(exp)) {                                                   \
                printf("%s: assertion %s failed line %u function %s!"   \
                " halted\n", SC_NAME(sc), #exp, __LINE__, __func__);\
                slhci_halt(sc, spipe, xfer);                            \
                ext;                                                    \
        }                                                               \
} while (/*CONSTCOND*/0)
#define UL_SLASSERT(exp, sc, spipe, xfer, ext) do {                     \
        if (!(exp)) {                                                   \
                printf("%s: assertion %s failed line %u function %s!"   \
                " halted\n", SC_NAME(sc), #exp, __LINE__, __func__);    \
                slhci_lock_call(sc, &slhci_halt, spipe, xfer);          \
                ext;                                                    \
        }                                                               \
} while (/*CONSTCOND*/0)
#else
#define LK_SLASSERT(exp, sc, spipe, xfer, ext) ((void)0)
#define UL_SLASSERT(exp, sc, spipe, xfer, ext) ((void)0)
#endif

const struct usbd_bus_methods slhci_bus_methods = {
        .ubm_open = slhci_open,
        .ubm_softint = slhci_void,
        .ubm_dopoll = slhci_poll,
        .ubm_allocx = slhci_allocx,
        .ubm_freex = slhci_freex,
        .ubm_getlock = slhci_get_lock,
        .ubm_rhctrl = slhci_roothub_ctrl,
};

const struct usbd_pipe_methods slhci_pipe_methods = {
        .upm_transfer = slhci_transfer,
        .upm_start = slhci_start,
        .upm_abort = slhci_abort,
        .upm_close = slhci_close,
        .upm_cleartoggle = slhci_clear_toggle,
        .upm_done = slhci_done,
};

const struct usbd_pipe_methods slhci_root_methods = {
        .upm_transfer = slhci_transfer,
        .upm_start = slhci_root_start,
        .upm_abort = slhci_abort,
        .upm_close = (void (*)(struct usbd_pipe *))slhci_void, /* XXX safe? */
        .upm_cleartoggle = slhci_clear_toggle,
        .upm_done = slhci_done,
};

/* Queue inlines */

#define GOT_FIRST_TO(tvar, t) \
    GCQ_GOT_FIRST_TYPED(tvar, &(t)->to, struct slhci_pipe, to)

#define FIND_TO(var, t, tvar, cond) \
    GCQ_FIND_TYPED(var, &(t)->to, tvar, struct slhci_pipe, to, cond)

#define FOREACH_AP(var, t, tvar) \
    GCQ_FOREACH_TYPED(var, &(t)->ap, tvar, struct slhci_pipe, ap)

#define GOT_FIRST_TIMED_COND(tvar, t, cond) \
    GCQ_GOT_FIRST_COND_TYPED(tvar, &(t)->timed, struct slhci_pipe, xq, cond)

#define GOT_FIRST_CB(tvar, t) \
    GCQ_GOT_FIRST_TYPED(tvar, &(t)->q[Q_CB], struct slhci_pipe, xq)

#define DEQUEUED_CALLBACK(tvar, t) \
    GCQ_DEQUEUED_FIRST_TYPED(tvar, &(t)->q[Q_CALLBACKS], struct slhci_pipe, xq)

#define FIND_TIMED(var, t, tvar, cond) \
   GCQ_FIND_TYPED(var, &(t)->timed, tvar, struct slhci_pipe, xq, cond)

#define DEQUEUED_WAITQ(tvar, sc) \
    GCQ_DEQUEUED_FIRST_TYPED(tvar, &(sc)->sc_waitq, struct slhci_pipe, xq)

static inline void
enter_waitq(struct slhci_softc *sc, struct slhci_pipe *spipe)
{
        gcq_insert_tail(&sc->sc_waitq, &spipe->xq);
}

static inline void
enter_q(struct slhci_transfers *t, struct slhci_pipe *spipe, int i)
{
        gcq_insert_tail(&t->q[i], &spipe->xq);
}

static inline void
enter_callback(struct slhci_transfers *t, struct slhci_pipe *spipe)
{
        gcq_insert_tail(&t->q[Q_CALLBACKS], &spipe->xq);
}

static inline void
enter_all_pipes(struct slhci_transfers *t, struct slhci_pipe *spipe)
{
        gcq_insert_tail(&t->ap, &spipe->ap);
}

/* Start out of lock functions. */

struct usbd_xfer *
slhci_allocx(struct usbd_bus *bus, unsigned int nframes)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct usbd_xfer *xfer;

        xfer = kmem_zalloc(sizeof(*xfer), KM_SLEEP);

        DLOG(D_MEM, "allocx %#jx", (uintptr_t)xfer, 0,0,0);

#ifdef SLHCI_MEM_ACCOUNTING
        slhci_mem_use(bus, 1);
#endif
#ifdef DIAGNOSTIC
        if (xfer != NULL)
                xfer->ux_state = XFER_BUSY;
#endif
        return xfer;
}

void
slhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        DLOG(D_MEM, "freex xfer %#jx spipe %#jx",
            (uintptr_t)xfer, (uintptr_t)xfer->ux_pipe,0,0);

#ifdef SLHCI_MEM_ACCOUNTING
        slhci_mem_use(bus, -1);
#endif
#ifdef DIAGNOSTIC
        if (xfer->ux_state != XFER_BUSY &&
            xfer->ux_status != USBD_NOT_STARTED) {
                struct slhci_softc *sc = SLHCI_BUS2SC(bus);
                printf("%s: slhci_freex: xfer=%p not busy, %#08x halted\n",
                    SC_NAME(sc), xfer, xfer->ux_state);
                DDOLOG("xfer=%p not busy, %#08x halted\n", xfer,
                    xfer->ux_state, 0, 0);
                slhci_lock_call(sc, &slhci_halt, NULL, NULL);
                return;
        }
        xfer->ux_state = XFER_FREE;
#endif

        kmem_free(xfer, sizeof(*xfer));
}

static void
slhci_get_lock(struct usbd_bus *bus, kmutex_t **lock)
{
        struct slhci_softc *sc = SLHCI_BUS2SC(bus);

        *lock = &sc->sc_lock;
}

usbd_status
slhci_transfer(struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        usbd_status error;

        DLOG(D_TRACE, "transfer type %jd xfer %#jx spipe %#jx ",
            SLHCI_XFER_TYPE(xfer), (uintptr_t)xfer, (uintptr_t)xfer->ux_pipe,
            0);

        /* Pipe isn't running, so start it first.  */
        error = xfer->ux_pipe->up_methods->upm_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));

        return error;
}

/* It is not safe for start to return anything other than USBD_INPROG. */
usbd_status
slhci_start(struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc = SLHCI_XFER2SC(xfer);
        struct usbd_pipe *pipe = xfer->ux_pipe;
        struct slhci_pipe *spipe = SLHCI_PIPE2SPIPE(pipe);
        struct slhci_transfers *t = &sc->sc_transfers;
        usb_endpoint_descriptor_t *ed = pipe->up_endpoint->ue_edesc;
        unsigned int max_packet;

        KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

        max_packet = UGETW(ed->wMaxPacketSize);

        DLOG(D_TRACE, "transfer type %jd start xfer %#jx spipe %#jx length %jd",
            spipe->ptype, (uintptr_t)xfer, (uintptr_t)spipe, xfer->ux_length);

        /* root transfers use slhci_root_start */

        KASSERT(spipe->xfer == NULL); /* not SLASSERT */

        xfer->ux_actlen = 0;
        xfer->ux_status = USBD_IN_PROGRESS;

        spipe->xfer = xfer;

        spipe->nerrs = 0;
        spipe->frame = t->frame;
        spipe->control = SL11_EPCTRL_ARM_ENABLE;
        spipe->tregs[DEV] = pipe->up_dev->ud_addr;
        spipe->tregs[PID] = spipe->newpid = UE_GET_ADDR(ed->bEndpointAddress)
            | (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN ? SL11_PID_IN :
            SL11_PID_OUT);
        spipe->newlen[0] = xfer->ux_length % max_packet;
        spipe->newlen[1] = uimin(xfer->ux_length, max_packet);

        if (spipe->ptype == PT_BULK || spipe->ptype == PT_INTR) {
                if (spipe->pflags & PF_TOGGLE)
                        spipe->control |= SL11_EPCTRL_DATATOGGLE;
                spipe->tregs[LEN] = spipe->newlen[1];
                if (spipe->tregs[LEN])
                        spipe->buffer = xfer->ux_buf;
                else
                        spipe->buffer = NULL;
                spipe->lastframe = t->frame;
                if (spipe->ptype == PT_INTR) {
                        spipe->frame = spipe->lastframe +
                            spipe->pipe.up_interval;
                }

#if defined(DEBUG) || defined(SLHCI_DEBUG)
                if (__predict_false(spipe->ptype == PT_INTR &&
                    xfer->ux_length > spipe->tregs[LEN])) {
                        printf("%s: Long INTR transfer not supported!\n",
                            SC_NAME(sc));
                        DDOLOG("Long INTR transfer not supported!", 0, 0, 0, 0);
                        xfer->ux_status = USBD_INVAL;
                }
#endif
        } else {
                /* ptype may be currently set to any control transfer type. */
                SLHCI_DEXEC(D_TRACE, slhci_log_xfer(xfer));

                /* SETUP contains IN/OUT bits also */
                spipe->tregs[PID] |= SL11_PID_SETUP;
                spipe->tregs[LEN] = 8;
                spipe->buffer = (uint8_t *)&xfer->ux_request;
                DLOGBUF(D_XFER, spipe->buffer, spipe->tregs[LEN]);
                spipe->ptype = PT_CTRL_SETUP;
                spipe->newpid &= ~SL11_PID_BITS;
                if (xfer->ux_length == 0 ||
                    (xfer->ux_request.bmRequestType & UT_READ))
                        spipe->newpid |= SL11_PID_IN;
                else
                        spipe->newpid |= SL11_PID_OUT;
        }

        if (xfer->ux_flags & USBD_FORCE_SHORT_XFER &&
            spipe->tregs[LEN] == max_packet &&
            (spipe->newpid & SL11_PID_BITS) == SL11_PID_OUT)
                spipe->wantshort = 1;
        else
                spipe->wantshort = 0;

        /*
         * The goal of newbustime and newlen is to avoid bustime calculation
         * in the interrupt.  The calculations are not too complex, but they
         * complicate the conditional logic somewhat and doing them all in the
         * same place shares constants. Index 0 is "short length" for bulk and
         * ctrl data and 1 is "full length" for ctrl data (bulk/intr are
         * already set to full length).
         */
        if (spipe->pflags & PF_LS) {
                /*
                 * Setting PREAMBLE for directly connected LS devices will
                 * lock up the chip.
                 */
                if (spipe->pflags & PF_PREAMBLE)
                        spipe->control |= SL11_EPCTRL_PREAMBLE;
                if (max_packet <= 8) {
                        spipe->bustime = SLHCI_LS_CONST +
                            SLHCI_LS_DATA_TIME(spipe->tregs[LEN]);
                        spipe->newbustime[0] = SLHCI_LS_CONST +
                            SLHCI_LS_DATA_TIME(spipe->newlen[0]);
                        spipe->newbustime[1] = SLHCI_LS_CONST +
                            SLHCI_LS_DATA_TIME(spipe->newlen[1]);
                } else
                        xfer->ux_status = USBD_INVAL;
        } else {
                UL_SLASSERT(pipe->up_dev->ud_speed == USB_SPEED_FULL, sc,
                    spipe, xfer, return USBD_IN_PROGRESS);
                if (max_packet <= SL11_MAX_PACKET_SIZE) {
                        spipe->bustime = SLHCI_FS_CONST +
                            SLHCI_FS_DATA_TIME(spipe->tregs[LEN]);
                        spipe->newbustime[0] = SLHCI_FS_CONST +
                            SLHCI_FS_DATA_TIME(spipe->newlen[0]);
                        spipe->newbustime[1] = SLHCI_FS_CONST +
                            SLHCI_FS_DATA_TIME(spipe->newlen[1]);
                } else
                        xfer->ux_status = USBD_INVAL;
        }

        /*
         * The datasheet incorrectly indicates that DIRECTION is for
         * "transmit to host".  It is for OUT and SETUP.  The app note
         * describes its use correctly.
         */
        if ((spipe->tregs[PID] & SL11_PID_BITS) != SL11_PID_IN)
                spipe->control |= SL11_EPCTRL_DIRECTION;

        slhci_start_entry(sc, spipe);

        return USBD_IN_PROGRESS;
}

usbd_status
slhci_root_start(struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc;
        struct slhci_pipe *spipe __diagused;

        spipe = SLHCI_PIPE2SPIPE(xfer->ux_pipe);
        sc = SLHCI_XFER2SC(xfer);

        struct slhci_transfers *t = &sc->sc_transfers;

        LK_SLASSERT(spipe != NULL && xfer != NULL, sc, spipe, xfer, return
            USBD_CANCELLED);

        DLOG(D_TRACE, "transfer type %jd start",
            SLHCI_XFER_TYPE(xfer), 0, 0, 0);

        KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));

        KASSERT(spipe->ptype == PT_ROOT_INTR);

        KASSERT(t->rootintr == NULL);
        t->rootintr = xfer;
        xfer->ux_status = USBD_IN_PROGRESS;

        return USBD_IN_PROGRESS;
}

usbd_status
slhci_open(struct usbd_pipe *pipe)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct usbd_device *dev;
        struct slhci_softc *sc;
        struct slhci_pipe *spipe;
        usb_endpoint_descriptor_t *ed;
        unsigned int max_packet, pmaxpkt;
        uint8_t rhaddr;

        dev = pipe->up_dev;
        sc = SLHCI_PIPE2SC(pipe);
        spipe = SLHCI_PIPE2SPIPE(pipe);
        ed = pipe->up_endpoint->ue_edesc;
        rhaddr = dev->ud_bus->ub_rhaddr;

        DLOG(D_TRACE, "slhci_open(addr=%jd,ep=%jd,rootaddr=%jd)",
                dev->ud_addr, ed->bEndpointAddress, rhaddr, 0);

        spipe->pflags = 0;
        spipe->frame = 0;
        spipe->lastframe = 0;
        spipe->xfer = NULL;
        spipe->buffer = NULL;

        gcq_init(&spipe->ap);
        gcq_init(&spipe->to);
        gcq_init(&spipe->xq);

        /*
         * The endpoint descriptor will not have been set up yet in the case
         * of the standard control pipe, so the max packet checks are also
         * necessary in start.
         */

        max_packet = UGETW(ed->wMaxPacketSize);

        if (dev->ud_speed == USB_SPEED_LOW) {
                spipe->pflags |= PF_LS;
                if (dev->ud_myhub->ud_addr != rhaddr) {
                        spipe->pflags |= PF_PREAMBLE;
                        if (!slhci_try_lsvh)
                                return slhci_lock_call(sc, &slhci_lsvh_warn,
                                    spipe, NULL);
                }
                pmaxpkt = 8;
        } else
                pmaxpkt = SL11_MAX_PACKET_SIZE;

        if (max_packet > pmaxpkt) {
                DLOG(D_ERR, "packet too large! size %jd spipe %#jx", max_packet,
                    (uintptr_t)spipe, 0,0);
                return USBD_INVAL;
        }

        if (dev->ud_addr == rhaddr) {
                switch (ed->bEndpointAddress) {
                case USB_CONTROL_ENDPOINT:
                        spipe->ptype = PT_ROOT_CTRL;
                        pipe->up_interval = 0;
                        pipe->up_methods = &roothub_ctrl_methods;
                        break;
                case UE_DIR_IN | USBROOTHUB_INTR_ENDPT:
                        spipe->ptype = PT_ROOT_INTR;
                        pipe->up_interval = 1;
                        pipe->up_methods = &slhci_root_methods;
                        break;
                default:
                        printf("%s: Invalid root endpoint!\n", SC_NAME(sc));
                        DDOLOG("Invalid root endpoint", 0, 0, 0, 0);
                        return USBD_INVAL;
                }
                return USBD_NORMAL_COMPLETION;
        } else {
                switch (ed->bmAttributes & UE_XFERTYPE) {
                case UE_CONTROL:
                        spipe->ptype = PT_CTRL_SETUP;
                        pipe->up_interval = 0;
                        break;
                case UE_INTERRUPT:
                        spipe->ptype = PT_INTR;
                        if (pipe->up_interval == USBD_DEFAULT_INTERVAL)
                                pipe->up_interval = ed->bInterval;
                        break;
                case UE_ISOCHRONOUS:
                        return slhci_lock_call(sc, &slhci_isoc_warn, spipe,
                            NULL);
                case UE_BULK:
                        spipe->ptype = PT_BULK;
                        pipe->up_interval = 0;
                        break;
                }

                DLOG(D_MSG, "open pipe type %jd interval %jd", spipe->ptype,
                    pipe->up_interval, 0,0);

                pipe->up_methods = __UNCONST(&slhci_pipe_methods);

                return slhci_lock_call(sc, &slhci_open_pipe, spipe, NULL);
        }
}

int
slhci_supported_rev(uint8_t rev)
{
        return rev >= SLTYPE_SL811HS_R12 && rev <= SLTYPE_SL811HS_R15;
}

/*
 * Must be called before the ISR is registered. Interrupts can be shared so
 * slhci_intr could be called as soon as the ISR is registered.
 * Note max_current argument is actual current, but stored as current/2
 */
void
slhci_preinit(struct slhci_softc *sc, PowerFunc pow, bus_space_tag_t iot,
    bus_space_handle_t ioh, uint16_t max_current, uint32_t stride)
{
        struct slhci_transfers *t;
        int i;

        t = &sc->sc_transfers;

#ifdef SLHCI_DEBUG
        ssc = sc;
#endif

        mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
        mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB);

        /* sc->sc_ier = 0;      */
        /* t->rootintr = NULL;  */
        t->flags = F_NODEV|F_UDISABLED;
        t->pend = INT_MAX;
        KASSERT(slhci_wait_time != INT_MAX);
        t->len[0] = t->len[1] = -1;
        if (max_current > 500)
                max_current = 500;
        t->max_current = (uint8_t)(max_current / 2);
        sc->sc_enable_power = pow;
        sc->sc_iot = iot;
        sc->sc_ioh = ioh;
        sc->sc_stride = stride;

        KASSERT(Q_MAX+1 == sizeof(t->q) / sizeof(t->q[0]));

        for (i = 0; i <= Q_MAX; i++)
                gcq_init_head(&t->q[i]);
        gcq_init_head(&t->timed);
        gcq_init_head(&t->to);
        gcq_init_head(&t->ap);
        gcq_init_head(&sc->sc_waitq);
}

int
slhci_attach(struct slhci_softc *sc)
{
        struct slhci_transfers *t;
        const char *rev;

        t = &sc->sc_transfers;

        /* Detect and check the controller type */
        t->sltype = SL11_GET_REV(slhci_read(sc, SL11_REV));

        /* SL11H not supported */
        if (!slhci_supported_rev(t->sltype)) {
                if (t->sltype == SLTYPE_SL11H)
                        printf("%s: SL11H unsupported or bus error!\n",
                            SC_NAME(sc));
                else
                        printf("%s: Unknown chip revision!\n", SC_NAME(sc));
                return -1;
        }

#ifdef SLHCI_DEBUG
        if (slhci_memtest(sc)) {
                printf("%s: memory/bus error!\n", SC_NAME(sc));
                return -1;
        }
#endif

        callout_init(&sc->sc_timer, CALLOUT_MPSAFE);
        callout_setfunc(&sc->sc_timer, slhci_reset_entry, sc);

        /*
         * It is not safe to call the soft interrupt directly as
         * usb_schedsoftintr does in the ub_usepolling case (due to locking).
         */
        sc->sc_cb_softintr = softint_establish(SOFTINT_NET,
            slhci_callback_entry, sc);

        if (t->sltype == SLTYPE_SL811HS_R12)
                rev = "(rev 1.2)";
        else if (t->sltype == SLTYPE_SL811HS_R14)
                rev = "(rev 1.4 or 1.5)";
        else
                rev = "(unknown revision)";

        aprint_normal("%s: ScanLogic SL811HS/T USB Host Controller %s\n",
            SC_NAME(sc), rev);

        aprint_normal("%s: Max Current %u mA (value by code, not by probe)\n",
            SC_NAME(sc), t->max_current * 2);

#if defined(SLHCI_DEBUG) || defined(SLHCI_NO_OVERTIME) || \
    defined(SLHCI_TRY_LSVH) || defined(SLHCI_PROFILE_TRANSFER)
        aprint_normal("%s: driver options:"
#ifdef SLHCI_DEBUG
        " SLHCI_DEBUG"
#endif
#ifdef SLHCI_TRY_LSVH
        " SLHCI_TRY_LSVH"
#endif
#ifdef SLHCI_NO_OVERTIME
        " SLHCI_NO_OVERTIME"
#endif
#ifdef SLHCI_PROFILE_TRANSFER
        " SLHCI_PROFILE_TRANSFER"
#endif
        "\n", SC_NAME(sc));
#endif
        sc->sc_bus.ub_revision = USBREV_1_1;
        sc->sc_bus.ub_methods = __UNCONST(&slhci_bus_methods);
        sc->sc_bus.ub_pipesize = sizeof(struct slhci_pipe);
        sc->sc_bus.ub_usedma = false;

        if (!sc->sc_enable_power)
                t->flags |= F_REALPOWER;

        t->flags |= F_ACTIVE;

        /* Attach usb and uhub. */
        sc->sc_child = config_found(SC_DEV(sc), &sc->sc_bus, usbctlprint,
            CFARGS_NONE);

        if (!sc->sc_child)
                return -1;
        else
                return 0;
}

int
slhci_detach(struct slhci_softc *sc, int flags)
{
        struct slhci_transfers *t;
        int ret;

        t = &sc->sc_transfers;

        /* By this point bus access is no longer allowed. */

        KASSERT(!(t->flags & F_ACTIVE));

        /*
         * To be MPSAFE is not sufficient to cancel callouts and soft
         * interrupts and assume they are dead since the code could already be
         * running or about to run.  Wait until they are known to be done.
         */
        while (t->flags & (F_RESET|F_CALLBACK))
                tsleep(&sc, PPAUSE, "slhci_detach", hz);

        softint_disestablish(sc->sc_cb_softintr);

        mutex_destroy(&sc->sc_lock);
        mutex_destroy(&sc->sc_intr_lock);

        ret = 0;

        if (sc->sc_child)
                ret = config_detach(sc->sc_child, flags);

#ifdef SLHCI_MEM_ACCOUNTING
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        if (sc->sc_mem_use) {
                printf("%s: Memory still in use after detach! mem_use (count)"
                    " = %d\n", SC_NAME(sc), sc->sc_mem_use);
                DDOLOG("Memory still in use after detach! mem_use (count)"
                    " = %d", sc->sc_mem_use, 0, 0, 0);
        }
#endif

        return ret;
}

int
slhci_activate(device_t self, enum devact act)
{
        struct slhci_softc *sc = device_private(self);

        switch (act) {
        case DVACT_DEACTIVATE:
                slhci_lock_call(sc, &slhci_halt, NULL, NULL);
                return 0;
        default:
                return EOPNOTSUPP;
        }
}

void
slhci_abort(struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc;
        struct slhci_pipe *spipe;

        spipe = SLHCI_PIPE2SPIPE(xfer->ux_pipe);

        if (spipe == NULL)
                goto callback;

        sc = SLHCI_XFER2SC(xfer);
        KASSERT(mutex_owned(&sc->sc_lock));

        DLOG(D_TRACE, "transfer type %jd abort xfer %#jx spipe %#jx "
            " spipe->xfer %#jx", spipe->ptype, (uintptr_t)xfer,
            (uintptr_t)spipe, (uintptr_t)spipe->xfer);

        slhci_lock_call(sc, &slhci_do_abort, spipe, xfer);

callback:
        xfer->ux_status = USBD_CANCELLED;
        usb_transfer_complete(xfer);
}

void
slhci_close(struct usbd_pipe *pipe)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc;
        struct slhci_pipe *spipe;

        sc = SLHCI_PIPE2SC(pipe);
        spipe = SLHCI_PIPE2SPIPE(pipe);

        DLOG(D_TRACE, "transfer type %jd close spipe %#jx spipe->xfer %#jx",
            spipe->ptype, (uintptr_t)spipe, (uintptr_t)spipe->xfer, 0);

        slhci_lock_call(sc, &slhci_close_pipe, spipe, NULL);
}

void
slhci_clear_toggle(struct usbd_pipe *pipe)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_pipe *spipe;

        spipe = SLHCI_PIPE2SPIPE(pipe);

        DLOG(D_TRACE, "transfer type %jd toggle spipe %#jx", spipe->ptype,
            (uintptr_t)spipe, 0, 0);

        spipe->pflags &= ~PF_TOGGLE;

#ifdef DIAGNOSTIC
        if (spipe->xfer != NULL) {
                struct slhci_softc *sc = (struct slhci_softc
                    *)pipe->up_dev->ud_bus;

                printf("%s: Clear toggle on transfer in progress! halted\n",
                    SC_NAME(sc));
                DDOLOG("Clear toggle on transfer in progress! halted",
                    0, 0, 0, 0);
                slhci_halt(sc, NULL, NULL);
        }
#endif
}

void
slhci_poll(struct usbd_bus *bus) /* XXX necessary? */
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc;

        sc = SLHCI_BUS2SC(bus);

        DLOG(D_TRACE, "slhci_poll", 0,0,0,0);

        slhci_lock_call(sc, &slhci_do_poll, NULL, NULL);
}

void
slhci_done(struct usbd_xfer *xfer)
{
}

void
slhci_void(void *v) {}

/* End out of lock functions. Start lock entry functions. */

#ifdef SLHCI_MEM_ACCOUNTING
void
slhci_mem_use(struct usbd_bus *bus, int val)
{
        struct slhci_softc *sc = SLHCI_BUS2SC(bus);

        mutex_enter(&sc->sc_intr_lock);
        sc->sc_mem_use += val;
        mutex_exit(&sc->sc_intr_lock);
}
#endif

void
slhci_reset_entry(void *arg)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc = arg;

        mutex_enter(&sc->sc_intr_lock);
        slhci_reset(sc);
        /*
         * We cannot call the callback directly since we could then be reset
         * again before finishing and need the callout delay for timing.
         * Scheduling the callout again before we exit would defeat the reap
         * mechanism since we could be unlocked while the reset flag is not
         * set. The callback code will check the wait queue.
         */
        slhci_callback_schedule(sc);
        mutex_exit(&sc->sc_intr_lock);
}

usbd_status
slhci_lock_call(struct slhci_softc *sc, LockCallFunc lcf, struct slhci_pipe
    *spipe, struct usbd_xfer *xfer)
{
        usbd_status ret;

        mutex_enter(&sc->sc_intr_lock);
        ret = (*lcf)(sc, spipe, xfer);
        slhci_main(sc);
        mutex_exit(&sc->sc_intr_lock);

        return ret;
}

void
slhci_start_entry(struct slhci_softc *sc, struct slhci_pipe *spipe)
{
        struct slhci_transfers *t;

        mutex_enter(&sc->sc_intr_lock);
        t = &sc->sc_transfers;

        if (!(t->flags & (F_AINPROG|F_BINPROG))) {
                slhci_enter_xfer(sc, spipe);
                slhci_dotransfer(sc);
                slhci_main(sc);
        } else {
                enter_waitq(sc, spipe);
        }
        mutex_exit(&sc->sc_intr_lock);
}

void
slhci_callback_entry(void *arg)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc;
        struct slhci_transfers *t;

        sc = (struct slhci_softc *)arg;

        mutex_enter(&sc->sc_intr_lock);
        t = &sc->sc_transfers;
        DLOG(D_SOFT, "callback_entry flags %#jx", t->flags, 0,0,0);

repeat:
        slhci_callback(sc);

        if (!gcq_empty(&sc->sc_waitq)) {
                slhci_enter_xfers(sc);
                slhci_dotransfer(sc);
                slhci_waitintr(sc, 0);
                goto repeat;
        }

        t->flags &= ~F_CALLBACK;
        mutex_exit(&sc->sc_intr_lock);
}

void
slhci_do_callback(struct slhci_softc *sc, struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        KASSERT(mutex_owned(&sc->sc_intr_lock));

        start_cc_time(&t_callback, (u_int)xfer);
        mutex_exit(&sc->sc_intr_lock);

        mutex_enter(&sc->sc_lock);
        usb_transfer_complete(xfer);
        mutex_exit(&sc->sc_lock);

        mutex_enter(&sc->sc_intr_lock);
        stop_cc_time(&t_callback);
}

int
slhci_intr(void *arg)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc = arg;
        int ret = 0;
        int irq;

        start_cc_time(&t_hard_int, (unsigned int)arg);
        mutex_enter(&sc->sc_intr_lock);

        do {
                irq = slhci_dointr(sc);
                ret |= irq;
                slhci_main(sc);
        } while (irq);
        mutex_exit(&sc->sc_intr_lock);

        stop_cc_time(&t_hard_int);
        return ret;
}

/* called with interrupt lock only held. */
void
slhci_main(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

waitcheck:
        slhci_waitintr(sc, slhci_wait_time);

        /*
         * The direct call is needed in the ub_usepolling and disabled cases
         * since the soft interrupt is not available.  In the disabled case,
         * this code can be reached from the usb detach, after the reaping of
         * the soft interrupt.  That test could be !F_ACTIVE, but there is no
         * reason not to make the callbacks directly in the other DISABLED
         * cases.
         */
        if ((t->flags & F_ROOTINTR) || !gcq_empty(&t->q[Q_CALLBACKS])) {
                if (__predict_false(sc->sc_bus.ub_usepolling ||
                    t->flags & F_DISABLED))
                        slhci_callback(sc);
                else
                        slhci_callback_schedule(sc);
        }

        if (!gcq_empty(&sc->sc_waitq)) {
                slhci_enter_xfers(sc);
                slhci_dotransfer(sc);
                goto waitcheck;
        }
        DLOG(D_INTR, "... done", 0, 0, 0, 0);
}

/* End lock entry functions. Start in lock function. */

/* Register read/write routines and barriers. */
#ifdef SLHCI_BUS_SPACE_BARRIERS
#define BSB(a, b, c, d, e) bus_space_barrier(a, b, c, d, BUS_SPACE_BARRIER_ # e)
#define BSB_SYNC(a, b, c, d) bus_space_barrier(a, b, c, d, BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
#else /* now !SLHCI_BUS_SPACE_BARRIERS */
#define BSB(a, b, c, d, e) __USE(d)
#define BSB_SYNC(a, b, c, d)
#endif /* SLHCI_BUS_SPACE_BARRIERS */

static void
slhci_write(struct slhci_softc *sc, uint8_t addr, uint8_t data)
{
        bus_size_t paddr, pdata, pst, psz;
        bus_space_tag_t iot;
        bus_space_handle_t ioh;

        paddr = pst = 0;
        pdata = sc->sc_stride;
        psz = pdata * 2;
        iot = sc->sc_iot;
        ioh = sc->sc_ioh;

        bus_space_write_1(iot, ioh, paddr, addr);
        BSB(iot, ioh, pst, psz, WRITE_BEFORE_WRITE);
        bus_space_write_1(iot, ioh, pdata, data);
        BSB(iot, ioh, pst, psz, WRITE_BEFORE_WRITE);
}

static uint8_t
slhci_read(struct slhci_softc *sc, uint8_t addr)
{
        bus_size_t paddr, pdata, pst, psz;
        bus_space_tag_t iot;
        bus_space_handle_t ioh;
        uint8_t data;

        paddr = pst = 0;
        pdata = sc->sc_stride;
        psz = pdata * 2;
        iot = sc->sc_iot;
        ioh = sc->sc_ioh;

        bus_space_write_1(iot, ioh, paddr, addr);
        BSB(iot, ioh, pst, psz, WRITE_BEFORE_READ);
        data = bus_space_read_1(iot, ioh, pdata);
        BSB(iot, ioh, pst, psz, READ_BEFORE_WRITE);
        return data;
}

#if 0 /* auto-increment mode broken, see errata doc */
static void
slhci_write_multi(struct slhci_softc *sc, uint8_t addr, uint8_t *buf, int l)
{
        bus_size_t paddr, pdata, pst, psz;
        bus_space_tag_t iot;
        bus_space_handle_t ioh;

        paddr = pst = 0;
        pdata = sc->sc_stride;
        psz = pdata * 2;
        iot = sc->sc_iot;
        ioh = sc->sc_ioh;

        bus_space_write_1(iot, ioh, paddr, addr);
        BSB(iot, ioh, pst, psz, WRITE_BEFORE_WRITE);
        bus_space_write_multi_1(iot, ioh, pdata, buf, l);
        BSB(iot, ioh, pst, psz, WRITE_BEFORE_WRITE);
}

static void
slhci_read_multi(struct slhci_softc *sc, uint8_t addr, uint8_t *buf, int l)
{
        bus_size_t paddr, pdata, pst, psz;
        bus_space_tag_t iot;
        bus_space_handle_t ioh;

        paddr = pst = 0;
        pdata = sc->sc_stride;
        psz = pdata * 2;
        iot = sc->sc_iot;
        ioh = sc->sc_ioh;

        bus_space_write_1(iot, ioh, paddr, addr);
        BSB(iot, ioh, pst, psz, WRITE_BEFORE_READ);
        bus_space_read_multi_1(iot, ioh, pdata, buf, l);
        BSB(iot, ioh, pst, psz, READ_BEFORE_WRITE);
}
#else
static void
slhci_write_multi(struct slhci_softc *sc, uint8_t addr, uint8_t *buf, int l)
{
#if 1
        for (; l; addr++, buf++, l--)
                slhci_write(sc, addr, *buf);
#else
        bus_size_t paddr, pdata, pst, psz;
        bus_space_tag_t iot;
        bus_space_handle_t ioh;

        paddr = pst = 0;
        pdata = sc->sc_stride;
        psz = pdata * 2;
        iot = sc->sc_iot;
        ioh = sc->sc_ioh;

        for (; l; addr++, buf++, l--) {
                bus_space_write_1(iot, ioh, paddr, addr);
                BSB(iot, ioh, pst, psz, WRITE_BEFORE_WRITE);
                bus_space_write_1(iot, ioh, pdata, *buf);
                BSB(iot, ioh, pst, psz, WRITE_BEFORE_WRITE);
        }
#endif
}

static void
slhci_read_multi(struct slhci_softc *sc, uint8_t addr, uint8_t *buf, int l)
{
#if 1
        for (; l; addr++, buf++, l--)
                *buf = slhci_read(sc, addr);
#else
        bus_size_t paddr, pdata, pst, psz;
        bus_space_tag_t iot;
        bus_space_handle_t ioh;

        paddr = pst = 0;
        pdata = sc->sc_stride;
        psz = pdata * 2;
        iot = sc->sc_iot;
        ioh = sc->sc_ioh;

        for (; l; addr++, buf++, l--) {
                bus_space_write_1(iot, ioh, paddr, addr);
                BSB(iot, ioh, pst, psz, WRITE_BEFORE_READ);
                *buf = bus_space_read_1(iot, ioh, pdata);
                BSB(iot, ioh, pst, psz, READ_BEFORE_WRITE);
        }
#endif
}
#endif

/*
 * After calling waitintr it is necessary to either call slhci_callback or
 * schedule the callback if necessary.  The callback cannot be called directly
 * from the hard interrupt since it interrupts at a high IPL and callbacks
 * can do copyout and such.
 */
static void
slhci_waitintr(struct slhci_softc *sc, int wait_time)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (__predict_false(sc->sc_bus.ub_usepolling))
                wait_time = 12000;

        while (t->pend <= wait_time) {
                DLOG(D_WAIT, "waiting... frame %jd pend %jd flags %#jx",
                    t->frame, t->pend, t->flags, 0);
                LK_SLASSERT(t->flags & F_ACTIVE, sc, NULL, NULL, return);
                LK_SLASSERT(t->flags & (F_AINPROG|F_BINPROG), sc, NULL, NULL,
                    return);
                slhci_dointr(sc);
        }
        DLOG(D_WAIT, "... done", 0, 0, 0, 0);
}

static int
slhci_dointr(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        struct slhci_pipe *tosp;
        uint8_t r;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (sc->sc_ier == 0) {
                DLOG(D_INTR, "sc_ier is zero", 0, 0, 0, 0);
                return 0;
        }

        r = slhci_read(sc, SL11_ISR);

#ifdef SLHCI_DEBUG
        if (slhcidebug & SLHCI_D_INTR && r & sc->sc_ier &&
            ((r & ~(SL11_ISR_SOF|SL11_ISR_DATA)) || slhcidebug & SLHCI_D_SOF)) {
                uint8_t e, f;

                e = slhci_read(sc, SL11_IER);
                f = slhci_read(sc, SL11_CTRL);
                DDOLOG("Flags=%#x IER=%#x ISR=%#x CTRL=%#x", t->flags, e, r, f);
                DDOLOGCTRL(f);
                DDOLOGISR(r);
        }
#endif

        /*
         * check IER for corruption occasionally.  Assume that the above
         * sc_ier == 0 case works correctly.
         */
        if (__predict_false(sc->sc_ier_check++ > SLHCI_IER_CHECK_FREQUENCY)) {
                sc->sc_ier_check = 0;
                if (sc->sc_ier != slhci_read(sc, SL11_IER)) {
                        printf("%s: IER value corrupted! halted\n",
                            SC_NAME(sc));
                        DDOLOG("IER value corrupted! halted", 0, 0, 0, 0);
                        slhci_halt(sc, NULL, NULL);
                        return 1;
                }
        }

        r &= sc->sc_ier;

        if (r == 0) {
                DLOG(D_INTR, "r is zero", 0, 0, 0, 0);
                return 0;
        }

        sc->sc_ier_check = 0;

        slhci_write(sc, SL11_ISR, r);
        BSB_SYNC(sc->iot, sc->ioh, sc->pst, sc->psz);

        /* If we have an insertion event we do not care about anything else. */
        if (__predict_false(r & SL11_ISR_INSERT)) {
                slhci_insert(sc);
                DLOG(D_INTR, "... done", 0, 0, 0, 0);
                return 1;
        }

        stop_cc_time(&t_intr);
        start_cc_time(&t_intr, r);

        if (r & SL11_ISR_SOF) {
                t->frame++;

                gcq_merge_tail(&t->q[Q_CB], &t->q[Q_NEXT_CB]);

                /*
                 * SOFCHECK flags are cleared in tstart.  Two flags are needed
                 * since the first SOF interrupt processed after the transfer
                 * is started might have been generated before the transfer
                 * was started.
                 */
                if (__predict_false(t->flags & F_SOFCHECK2 && t->flags &
                    (F_AINPROG|F_BINPROG))) {
                        printf("%s: Missed transfer completion. halted\n",
                            SC_NAME(sc));
                        DDOLOG("Missed transfer completion. halted", 0, 0, 0,
                            0);
                        slhci_halt(sc, NULL, NULL);
                        return 1;
                } else if (t->flags & F_SOFCHECK1) {
                        t->flags |= F_SOFCHECK2;
                } else
                        t->flags |= F_SOFCHECK1;

                if (t->flags & F_CHANGE)
                        t->flags |= F_ROOTINTR;

                while (__predict_true(GOT_FIRST_TO(tosp, t)) &&
                    __predict_false(tosp->to_frame <= t->frame)) {
                        tosp->xfer->ux_status = USBD_TIMEOUT;
                        slhci_do_abort(sc, tosp, tosp->xfer);
                        enter_callback(t, tosp);
                }

                /*
                 * Start any waiting transfers right away.  If none, we will
                 * start any new transfers later.
                 */
                slhci_tstart(sc);
        }

        if (r & (SL11_ISR_USBA|SL11_ISR_USBB)) {
                int ab;

                if ((r & (SL11_ISR_USBA|SL11_ISR_USBB)) ==
                    (SL11_ISR_USBA|SL11_ISR_USBB)) {
                        if (!(t->flags & (F_AINPROG|F_BINPROG)))
                                return 1; /* presume card pulled */

                        LK_SLASSERT((t->flags & (F_AINPROG|F_BINPROG)) !=
                            (F_AINPROG|F_BINPROG), sc, NULL, NULL, return 1);

                        /*
                         * This should never happen (unless card removal just
                         * occurred) but appeared frequently when both
                         * transfers were started at the same time and was
                         * accompanied by data corruption.  It still happens
                         * at times.  I have not seen data correption except
                         * when the STATUS bit gets set, which now causes the
                         * driver to halt, however this should still not
                         * happen so the warning is kept.  See comment in
                         * abdone, below.
                         */
                        printf("%s: Transfer reported done but not started! "
                            "Verify data integrity if not detaching. "
                            " flags %#x r %x\n", SC_NAME(sc), t->flags, r);

                        if (!(t->flags & F_AINPROG))
                                r &= ~SL11_ISR_USBA;
                        else
                                r &= ~SL11_ISR_USBB;
                }
                t->pend = INT_MAX;

                if (r & SL11_ISR_USBA)
                        ab = A;
                else
                        ab = B;

                /*
                 * This happens when a low speed device is attached to
                 * a hub with chip rev 1.5.  SOF stops, but a few transfers
                 * still work before causing this error.
                 */
                if (!(t->flags & (ab ? F_BINPROG : F_AINPROG))) {
                        printf("%s: %s done but not in progress! halted\n",
                            SC_NAME(sc), ab ? "B" : "A");
                        DDOLOG("AB=%d done but not in progress! halted", ab,
                            0, 0, 0);
                        slhci_halt(sc, NULL, NULL);
                        return 1;
                }

                t->flags &= ~(ab ? F_BINPROG : F_AINPROG);
                slhci_tstart(sc);
                stop_cc_time(&t_ab[ab]);
                start_cc_time(&t_abdone, t->flags);
                slhci_abdone(sc, ab);
                stop_cc_time(&t_abdone);
        }

        slhci_dotransfer(sc);

        DLOG(D_INTR, "... done", 0, 0, 0, 0);

        return 1;
}

static void
slhci_abdone(struct slhci_softc *sc, int ab)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        struct slhci_pipe *spipe;
        struct usbd_xfer *xfer;
        uint8_t status, buf_start;
        uint8_t *target_buf;
        unsigned int actlen;
        int head;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        DLOG(D_TRACE, "ABDONE flags %#jx", t->flags, 0,0,0);

        DLOG(D_MSG, "DONE AB=%jd spipe %#jx len %jd xfer %#jx", ab,
            t->spipe[ab], (uintptr_t)t->len[ab],
            (uintptr_t)(t->spipe[ab] ? t->spipe[ab]->xfer : NULL));

        spipe = t->spipe[ab];

        /*
         * skip this one if aborted; do not call return from the rest of the
         * function unless halting, else t->len will not be cleared.
         */
        if (spipe == NULL)
                goto done;

        t->spipe[ab] = NULL;

        xfer = spipe->xfer;

        gcq_remove(&spipe->to);

        LK_SLASSERT(xfer != NULL, sc, spipe, NULL, return);

        status = slhci_read(sc, slhci_tregs[ab][STAT]);

        /*
         * I saw no status or remaining length greater than the requested
         * length in early driver versions in circumstances I assumed caused
         * excess power draw.  I am no longer able to reproduce this when
         * causing excess power draw circumstances.
         *
         * Disabling a power check and attaching aue to a keyboard and hub
         * that is directly attached (to CFU1U, 100mA max, aue 160mA, keyboard
         * 98mA) sometimes works and sometimes fails to configure.  After
         * removing the aue and attaching a self-powered umass dvd reader
         * (unknown if it draws power from the host also) soon a single Error
         * status occurs then only timeouts. The controller soon halts freeing
         * memory due to being ONQU instead of BUSY.  This may be the same
         * basic sequence that caused the no status/bad length errors.  The
         * umass device seems to work (better at least) with the keyboard hub
         * when not first attaching aue (tested once reading an approximately
         * 200MB file).
         *
         * Overflow can indicate that the device and host disagree about how
         * much data has been transferred.  This may indicate a problem at any
         * point during the transfer, not just when the error occurs.  It may
         * indicate data corruption.  A warning message is printed.
         *
         * Trying to use both A and B transfers at the same time results in
         * incorrect transfer completion ISR reports and the status will then
         * include SL11_EPSTAT_SETUP, which is apparently set while the
         * transfer is in progress.  I also noticed data corruption, even
         * after waiting for the transfer to complete. The driver now avoids
         * trying to start both at the same time.
         *
         * I had accidently initialized the B registers before they were valid
         * in some driver versions.  Since every other performance enhancing
         * feature has been confirmed buggy in the errata doc, I have not
         * tried both transfers at once again with the documented
         * initialization order.
         *
         * However, I have seen this problem again ("done but not started"
         * errors), which in some cases cases the SETUP status bit to remain
         * set on future transfers.  In other cases, the SETUP bit is not set
         * and no data corruption occurs.  This occurred while using both umass
         * and aue on a powered hub (maybe triggered by some local activity
         * also) and needs several reads of the 200MB file to trigger.  The
         * driver now halts if SETUP is detected.
         */

        actlen = 0;

        if (__predict_false(!status)) {
                DDOLOG("no status! xfer %p spipe %p", xfer, spipe, 0,0);
                printf("%s: no status! halted\n", SC_NAME(sc));
                slhci_halt(sc, spipe, xfer);
                return;
        }

#ifdef SLHCI_DEBUG
        if ((slhcidebug & SLHCI_D_NAK) ||
            (status & SL11_EPSTAT_ERRBITS) != SL11_EPSTAT_NAK) {
                DDOLOG("USB Status = %#.2x", status, 0, 0, 0);
                DDOLOGSTATUS(status);
        }
#endif

        if (!(status & SL11_EPSTAT_ERRBITS)) {
                unsigned int cont = slhci_read(sc, slhci_tregs[ab][CONT]);
                unsigned int len = spipe->tregs[LEN];
                DLOG(D_XFER, "cont %jd len %jd", cont, len, 0, 0);
                if ((status & SL11_EPSTAT_OVERFLOW) || cont > len) {
                        DDOLOG("overflow - cont %d len %d xfer->ux_length %d "
                            "xfer->actlen %d", cont, len, xfer->ux_length,
                            xfer->ux_actlen);
                        printf("%s: overflow cont %d len %d xfer->ux_length"
                            " %d xfer->ux_actlen %d\n", SC_NAME(sc), cont,
                            len, xfer->ux_length, xfer->ux_actlen);
                        actlen = len;
                } else {
                        actlen = len - cont;
                }
                spipe->nerrs = 0;
        }

        /* Actual copyin done after starting next transfer. */
        if (actlen && (spipe->tregs[PID] & SL11_PID_BITS) == SL11_PID_IN) {
                target_buf = spipe->buffer;
                buf_start = spipe->tregs[ADR];
        } else {
                target_buf = NULL;
                buf_start = 0; /* XXX gcc uninitialized warnings */
        }

        if (status & SL11_EPSTAT_ERRBITS) {
                status &= SL11_EPSTAT_ERRBITS;
                if (status & SL11_EPSTAT_SETUP) {
                        printf("%s: Invalid controller state detected! "
                            "halted\n", SC_NAME(sc));
                        DDOLOG("Invalid controller state detected! "
                            "halted", 0, 0, 0, 0);
                        slhci_halt(sc, spipe, xfer);
                        return;
                } else if (__predict_false(sc->sc_bus.ub_usepolling)) {
                        head = Q_CALLBACKS;
                        if (status & SL11_EPSTAT_STALL)
                                xfer->ux_status = USBD_STALLED;
                        else if (status & SL11_EPSTAT_TIMEOUT)
                                xfer->ux_status = USBD_TIMEOUT;
                        else if (status & SL11_EPSTAT_NAK)
                                head = Q_NEXT_CB;
                        else
                                xfer->ux_status = USBD_IOERROR;
                } else if (status & SL11_EPSTAT_NAK) {
                        int i = spipe->pipe.up_interval;
                        if (i == 0)
                                i = 1;
                        DDOLOG("xfer %p spipe %p NAK delay by %d", xfer, spipe,
                            i, 0);
                        spipe->lastframe = spipe->frame = t->frame + i;
                        slhci_queue_timed(sc, spipe);
                        goto queued;
                } else if (++spipe->nerrs > SLHCI_MAX_RETRIES ||
                    (status & SL11_EPSTAT_STALL)) {
                        DDOLOG("xfer %p spipe %p nerrs %d", xfer, spipe,
                            spipe->nerrs, 0);
                        if (status & SL11_EPSTAT_STALL)
                                xfer->ux_status = USBD_STALLED;
                        else if (status & SL11_EPSTAT_TIMEOUT)
                                xfer->ux_status = USBD_TIMEOUT;
                        else
                                xfer->ux_status = USBD_IOERROR;

                        DLOG(D_ERR, "Max retries reached! status %#jx "
                            "xfer->ux_status %jd", status, xfer->ux_status, 0,
                            0);
                        DDOLOGSTATUS(status);

                        head = Q_CALLBACKS;
                } else {
                        head = Q_NEXT_CB;
                }
        } else if (spipe->ptype == PT_CTRL_SETUP) {
                spipe->tregs[PID] = spipe->newpid;

                if (xfer->ux_length) {
                        LK_SLASSERT(spipe->newlen[1] != 0, sc, spipe, xfer,
                            return);
                        spipe->tregs[LEN] = spipe->newlen[1];
                        spipe->bustime = spipe->newbustime[1];
                        spipe->buffer = xfer->ux_buf;
                        spipe->ptype = PT_CTRL_DATA;
                } else {
status_setup:
                        /* CTRL_DATA swaps direction in PID then jumps here */
                        spipe->tregs[LEN] = 0;
                        if (spipe->pflags & PF_LS)
                                spipe->bustime = SLHCI_LS_CONST;
                        else
                                spipe->bustime = SLHCI_FS_CONST;
                        spipe->ptype = PT_CTRL_STATUS;
                        spipe->buffer = NULL;
                }

                /* Status or first data packet must be DATA1. */
                spipe->control |= SL11_EPCTRL_DATATOGGLE;
                if ((spipe->tregs[PID] & SL11_PID_BITS) == SL11_PID_IN)
                        spipe->control &= ~SL11_EPCTRL_DIRECTION;
                else
                        spipe->control |= SL11_EPCTRL_DIRECTION;

                head = Q_CB;
        } else if (spipe->ptype == PT_CTRL_STATUS) {
                head = Q_CALLBACKS;
        } else { /* bulk, intr, control data */
                xfer->ux_actlen += actlen;
                spipe->control ^= SL11_EPCTRL_DATATOGGLE;

                if (actlen == spipe->tregs[LEN] &&
                    (xfer->ux_length > xfer->ux_actlen || spipe->wantshort)) {
                        spipe->buffer += actlen;
                        LK_SLASSERT(xfer->ux_length >= xfer->ux_actlen, sc,
                            spipe, xfer, return);
                        if (xfer->ux_length - xfer->ux_actlen < actlen) {
                                spipe->wantshort = 0;
                                spipe->tregs[LEN] = spipe->newlen[0];
                                spipe->bustime = spipe->newbustime[0];
                                LK_SLASSERT(xfer->ux_actlen +
                                    spipe->tregs[LEN] == xfer->ux_length, sc,
                                    spipe, xfer, return);
                        }
                        head = Q_CB;
                } else if (spipe->ptype == PT_CTRL_DATA) {
                        spipe->tregs[PID] ^= SLHCI_PID_SWAP_IN_OUT;
                        goto status_setup;
                } else {
                        if (spipe->ptype == PT_INTR) {
                                spipe->lastframe +=
                                    spipe->pipe.up_interval;
                                /*
                                 * If ack, we try to keep the
                                 * interrupt rate by using lastframe
                                 * instead of the current frame.
                                 */
                                spipe->frame = spipe->lastframe +
                                    spipe->pipe.up_interval;
                        }

                        /*
                         * Set the toggle for the next transfer.  It
                         * has already been toggled above, so the
                         * current setting will apply to the next
                         * transfer.
                         */
                        if (spipe->control & SL11_EPCTRL_DATATOGGLE)
                                spipe->pflags |= PF_TOGGLE;
                        else
                                spipe->pflags &= ~PF_TOGGLE;

                        head = Q_CALLBACKS;
                }
        }

        if (head == Q_CALLBACKS) {
                gcq_remove(&spipe->to);

                if (xfer->ux_status == USBD_IN_PROGRESS) {
                        LK_SLASSERT(xfer->ux_actlen <= xfer->ux_length, sc,
                            spipe, xfer, return);
                        xfer->ux_status = USBD_NORMAL_COMPLETION;
                }
        }

        enter_q(t, spipe, head);

queued:
        if (target_buf != NULL) {
                slhci_dotransfer(sc);
                start_cc_time(&t_copy_from_dev, actlen);
                slhci_read_multi(sc, buf_start, target_buf, actlen);
                stop_cc_time(&t_copy_from_dev);
                DLOGBUF(D_BUF, target_buf, actlen);
                t->pend -= SLHCI_FS_CONST + SLHCI_FS_DATA_TIME(actlen);
        }

done:
        t->len[ab] = -1;
}

static void
slhci_tstart(struct slhci_softc *sc)
{
        struct slhci_transfers *t;
        struct slhci_pipe *spipe;
        int remaining_bustime;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (!(t->flags & (F_AREADY|F_BREADY)))
                return;

        if (t->flags & (F_AINPROG|F_BINPROG|F_DISABLED))
                return;

        /*
         * We have about 6 us to get from the bus time check to
         * starting the transfer or we might babble or the chip might fail to
         * signal transfer complete.  This leaves no time for any other
         * interrupts.
         */
        remaining_bustime = (int)(slhci_read(sc, SL811_CSOF)) << 6;
        remaining_bustime -= SLHCI_END_BUSTIME;

        /*
         * Start one transfer only, clearing any aborted transfers that are
         * not yet in progress and skipping missed isoc. It is easier to copy
         * & paste most of the A/B sections than to make the logic work
         * otherwise and this allows better constant use.
         */
        if (t->flags & F_AREADY) {
                spipe = t->spipe[A];
                if (spipe == NULL) {
                        t->flags &= ~F_AREADY;
                        t->len[A] = -1;
                } else if (remaining_bustime >= spipe->bustime) {
                        t->flags &= ~(F_AREADY|F_SOFCHECK1|F_SOFCHECK2);
                        t->flags |= F_AINPROG;
                        start_cc_time(&t_ab[A], spipe->tregs[LEN]);
                        slhci_write(sc, SL11_E0CTRL, spipe->control);
                        goto pend;
                }
        }
        if (t->flags & F_BREADY) {
                spipe = t->spipe[B];
                if (spipe == NULL) {
                        t->flags &= ~F_BREADY;
                        t->len[B] = -1;
                } else if (remaining_bustime >= spipe->bustime) {
                        t->flags &= ~(F_BREADY|F_SOFCHECK1|F_SOFCHECK2);
                        t->flags |= F_BINPROG;
                        start_cc_time(&t_ab[B], spipe->tregs[LEN]);
                        slhci_write(sc, SL11_E1CTRL, spipe->control);
pend:
                        t->pend = spipe->bustime;
                }
        }
}

static void
slhci_dotransfer(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        struct slhci_pipe *spipe;
        int ab, i;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        while ((t->len[A] == -1 || t->len[B] == -1) &&
            (GOT_FIRST_TIMED_COND(spipe, t, spipe->frame <= t->frame) ||
            GOT_FIRST_CB(spipe, t))) {
                LK_SLASSERT(spipe->xfer != NULL, sc, spipe, NULL, return);
                LK_SLASSERT(spipe->ptype != PT_ROOT_CTRL && spipe->ptype !=
                    PT_ROOT_INTR, sc, spipe, NULL, return);

                /* Check that this transfer can fit in the remaining memory. */
                if (t->len[A] + t->len[B] + spipe->tregs[LEN] + 1 >
                    SL11_MAX_PACKET_SIZE) {
                        DLOG(D_XFER, "Transfer does not fit. alen %jd blen %jd "
                            "len %jd", t->len[A], t->len[B], spipe->tregs[LEN],
                            0);
                        return;
                }

                gcq_remove(&spipe->xq);

                if (t->len[A] == -1) {
                        ab = A;
                        spipe->tregs[ADR] = SL11_BUFFER_START;
                } else {
                        ab = B;
                        spipe->tregs[ADR] = SL11_BUFFER_END -
                            spipe->tregs[LEN];
                }

                t->len[ab] = spipe->tregs[LEN];

                if (spipe->tregs[LEN] && (spipe->tregs[PID] & SL11_PID_BITS)
                    != SL11_PID_IN) {
                        start_cc_time(&t_copy_to_dev,
                            spipe->tregs[LEN]);
                        slhci_write_multi(sc, spipe->tregs[ADR],
                            spipe->buffer, spipe->tregs[LEN]);
                        stop_cc_time(&t_copy_to_dev);
                        t->pend -= SLHCI_FS_CONST +
                            SLHCI_FS_DATA_TIME(spipe->tregs[LEN]);
                }

                DLOG(D_MSG, "NEW TRANSFER AB=%jd flags %#jx alen %jd blen %jd",
                    ab, t->flags, t->len[0], t->len[1]);

                if (spipe->tregs[LEN])
                        i = 0;
                else
                        i = 1;

                for (; i <= 3; i++)
                        if (t->current_tregs[ab][i] != spipe->tregs[i]) {
                                t->current_tregs[ab][i] = spipe->tregs[i];
                                slhci_write(sc, slhci_tregs[ab][i],
                                    spipe->tregs[i]);
                        }

                DLOG(D_SXFER, "Transfer len %jd pid %#jx dev %jd type %jd",
                    spipe->tregs[LEN], spipe->tregs[PID], spipe->tregs[DEV],
                    spipe->ptype);

                t->spipe[ab] = spipe;
                t->flags |= ab ? F_BREADY : F_AREADY;

                slhci_tstart(sc);
        }
}

/*
 * slhci_callback is called after the lock is taken.
 */
static void
slhci_callback(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        struct slhci_pipe *spipe;
        struct usbd_xfer *xfer;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        DLOG(D_SOFT, "CB flags %#jx", t->flags, 0,0,0);
        for (;;) {
                if (__predict_false(t->flags & F_ROOTINTR)) {
                        t->flags &= ~F_ROOTINTR;
                        if (t->rootintr != NULL) {
                                u_char *p;

                                KASSERT(t->rootintr->ux_status ==
                                    USBD_IN_PROGRESS);
                                p = t->rootintr->ux_buf;
                                p[0] = 2;
                                t->rootintr->ux_actlen = 1;
                                t->rootintr->ux_status = USBD_NORMAL_COMPLETION;
                                xfer = t->rootintr;
                                goto do_callback;
                        }
                }


                if (!DEQUEUED_CALLBACK(spipe, t))
                        return;

                xfer = spipe->xfer;
                LK_SLASSERT(xfer != NULL, sc, spipe, NULL, return);
                spipe->xfer = NULL;
                DLOG(D_XFER, "xfer callback length %jd actlen %jd spipe %#jx "
                    "type %jd", xfer->ux_length, (uintptr_t)xfer->ux_actlen,
                    (uintptr_t)spipe, spipe->ptype);
do_callback:
                slhci_do_callback(sc, xfer);
        }
}

static void
slhci_enter_xfer(struct slhci_softc *sc, struct slhci_pipe *spipe)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (__predict_false(t->flags & F_DISABLED) ||
            __predict_false(spipe->pflags & PF_GONE)) {
                DLOG(D_MSG, "slhci_enter_xfer: DISABLED or GONE", 0,0,0,0);
                spipe->xfer->ux_status = USBD_CANCELLED;
        }

        if (spipe->xfer->ux_status == USBD_IN_PROGRESS) {
                if (spipe->xfer->ux_timeout) {
                        spipe->to_frame = t->frame + spipe->xfer->ux_timeout;
                        slhci_xfer_timer(sc, spipe);
                }
                if (spipe->pipe.up_interval)
                        slhci_queue_timed(sc, spipe);
                else
                        enter_q(t, spipe, Q_CB);
        } else
                enter_callback(t, spipe);
}

static void
slhci_enter_xfers(struct slhci_softc *sc)
{
        struct slhci_pipe *spipe;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        while (DEQUEUED_WAITQ(spipe, sc))
                slhci_enter_xfer(sc, spipe);
}

static void
slhci_queue_timed(struct slhci_softc *sc, struct slhci_pipe *spipe)
{
        struct slhci_transfers *t;
        struct gcq *q;
        struct slhci_pipe *spp;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        FIND_TIMED(q, t, spp, spp->frame > spipe->frame);
        gcq_insert_before(q, &spipe->xq);
}

static void
slhci_xfer_timer(struct slhci_softc *sc, struct slhci_pipe *spipe)
{
        struct slhci_transfers *t;
        struct gcq *q;
        struct slhci_pipe *spp;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        FIND_TO(q, t, spp, spp->to_frame >= spipe->to_frame);
        gcq_insert_before(q, &spipe->to);
}

static void
slhci_callback_schedule(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (t->flags & F_ACTIVE)
                slhci_do_callback_schedule(sc);
}

static void
slhci_do_callback_schedule(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        DLOG(D_MSG, "flags %#jx", t->flags, 0, 0, 0);
        if (!(t->flags & F_CALLBACK)) {
                t->flags |= F_CALLBACK;
                softint_schedule(sc->sc_cb_softintr);
        }
}

#if 0
/* must be called with lock taken. */
/* XXX static */ void
slhci_pollxfer(struct slhci_softc *sc, struct usbd_xfer *xfer)
{
        KASSERT(mutex_owned(&sc->sc_intr_lock));
        slhci_dotransfer(sc);
        do {
                slhci_dointr(sc);
        } while (xfer->ux_status == USBD_IN_PROGRESS);
        slhci_do_callback(sc, xfer);
}
#endif

static usbd_status
slhci_do_poll(struct slhci_softc *sc, struct slhci_pipe *spipe, struct
    usbd_xfer *xfer)
{
        slhci_waitintr(sc, 0);

        return USBD_NORMAL_COMPLETION;
}

static usbd_status
slhci_lsvh_warn(struct slhci_softc *sc, struct slhci_pipe *spipe, struct
    usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        if (!(t->flags & F_LSVH_WARNED)) {
                printf("%s: Low speed device via hub disabled, "
                    "see slhci(4)\n", SC_NAME(sc));
                DDOLOG("Low speed device via hub disabled, "
                    "see slhci(4)", SC_NAME(sc), 0,0,0);
                t->flags |= F_LSVH_WARNED;
        }
        return USBD_INVAL;
}

static usbd_status
slhci_isoc_warn(struct slhci_softc *sc, struct slhci_pipe *spipe, struct
    usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        if (!(t->flags & F_ISOC_WARNED)) {
                printf("%s: ISOC transfer not supported "
                    "(see slhci(4))\n", SC_NAME(sc));
                DDOLOG("ISOC transfer not supported "
                    "(see slhci(4))", 0, 0, 0, 0);
                t->flags |= F_ISOC_WARNED;
        }
        return USBD_INVAL;
}

static usbd_status
slhci_open_pipe(struct slhci_softc *sc, struct slhci_pipe *spipe, struct
    usbd_xfer *xfer)
{
        struct slhci_transfers *t;
        struct usbd_pipe *pipe;

        t = &sc->sc_transfers;
        pipe = &spipe->pipe;

        if (t->flags & F_DISABLED)
                return USBD_CANCELLED;
        else if (pipe->up_interval && !slhci_reserve_bustime(sc, spipe, 1))
                return USBD_PENDING_REQUESTS;
        else {
                enter_all_pipes(t, spipe);
                return USBD_NORMAL_COMPLETION;
        }
}

static usbd_status
slhci_close_pipe(struct slhci_softc *sc, struct slhci_pipe *spipe, struct
    usbd_xfer *xfer)
{
        struct usbd_pipe *pipe;

        pipe = &spipe->pipe;

        if (pipe->up_interval && spipe->ptype != PT_ROOT_INTR)
                slhci_reserve_bustime(sc, spipe, 0);
        gcq_remove(&spipe->ap);
        return USBD_NORMAL_COMPLETION;
}

static usbd_status
slhci_do_abort(struct slhci_softc *sc, struct slhci_pipe *spipe, struct
    usbd_xfer *xfer)
{
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (spipe->xfer == xfer) {
                if (spipe->ptype == PT_ROOT_INTR) {
                        if (t->rootintr == spipe->xfer) /* XXX assert? */
                                t->rootintr = NULL;
                } else {
                        gcq_remove(&spipe->to);
                        gcq_remove(&spipe->xq);

                        if (t->spipe[A] == spipe) {
                                t->spipe[A] = NULL;
                                if (!(t->flags & F_AINPROG))
                                        t->len[A] = -1;
                        } else if (t->spipe[B] == spipe) {
                                        t->spipe[B] = NULL;
                                if (!(t->flags & F_BINPROG))
                                        t->len[B] = -1;
                        }
                }

                if (xfer->ux_status != USBD_TIMEOUT) {
                        spipe->xfer = NULL;
                        spipe->pipe.up_repeat = 0; /* XXX timeout? */
                }
        }

        return USBD_NORMAL_COMPLETION;
}

/*
 * Called to deactivate or stop use of the controller instead of panicking.
 * Will cancel the xfer correctly even when not on a list.
 */
static usbd_status
slhci_halt(struct slhci_softc *sc, struct slhci_pipe *spipe,
    struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        t = &sc->sc_transfers;

        DDOLOG("Halt! sc %p spipe %p xfer %p", sc, spipe, xfer, 0);

        if (spipe != NULL)
                slhci_log_spipe(spipe);

        if (xfer != NULL)
                slhci_log_xfer(xfer);

        if (spipe != NULL && xfer != NULL && spipe->xfer == xfer &&
            !gcq_onlist(&spipe->xq) && t->spipe[A] != spipe && t->spipe[B] !=
            spipe) {
                xfer->ux_status = USBD_CANCELLED;
                enter_callback(t, spipe);
        }

        if (t->flags & F_ACTIVE) {
                slhci_intrchange(sc, 0);
                /*
                 * leave power on when halting in case flash devices or disks
                 * are attached, which may be writing and could be damaged
                 * by abrupt power loss.  The root hub clear power feature
                 * should still work after halting.
                 */
        }

        t->flags &= ~F_ACTIVE;
        t->flags |= F_UDISABLED;
        if (!(t->flags & F_NODEV))
                t->flags |= F_NODEV|F_CCONNECT|F_ROOTINTR;
        slhci_drain(sc);

        /* One last callback for the drain and device removal. */
        slhci_do_callback_schedule(sc);

        return USBD_NORMAL_COMPLETION;
}

/*
 * There are three interrupt states: no interrupts during reset and after
 * device deactivation, INSERT only for no device present but power on, and
 * SOF, INSERT, ADONE, and BDONE when device is present.
 */
static void
slhci_intrchange(struct slhci_softc *sc, uint8_t new_ier)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        KASSERT(mutex_owned(&sc->sc_intr_lock));
        if (sc->sc_ier != new_ier) {
                DLOG(D_INTR, "New IER %#jx", new_ier, 0, 0, 0);
                sc->sc_ier = new_ier;
                slhci_write(sc, SL11_IER, new_ier);
                BSB_SYNC(sc->iot, sc->ioh, sc->pst, sc->psz);
        }
}

/*
 * Drain: cancel all pending transfers and put them on the callback list and
 * set the UDISABLED flag.  UDISABLED is cleared only by reset.
 */
static void
slhci_drain(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        struct slhci_pipe *spipe;
        struct gcq *q;
        int i;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        t = &sc->sc_transfers;

        DLOG(D_MSG, "DRAIN flags %#jx", t->flags, 0,0,0);

        t->pend = INT_MAX;

        for (i = 0; i <= 1; i++) {
                t->len[i] = -1;
                if (t->spipe[i] != NULL) {
                        enter_callback(t, t->spipe[i]);
                        t->spipe[i] = NULL;
                }
        }

        /* Merge the queues into the callback queue. */
        gcq_merge_tail(&t->q[Q_CALLBACKS], &t->q[Q_CB]);
        gcq_merge_tail(&t->q[Q_CALLBACKS], &t->q[Q_NEXT_CB]);
        gcq_merge_tail(&t->q[Q_CALLBACKS], &t->timed);

        /*
         * Cancel all pipes.  Note that not all of these may be on the
         * callback queue yet; some could be in slhci_start, for example.
         */
        FOREACH_AP(q, t, spipe) {
                spipe->pflags |= PF_GONE;
                spipe->pipe.up_repeat = 0;
                spipe->pipe.up_aborting = 1;
                if (spipe->xfer != NULL)
                        spipe->xfer->ux_status = USBD_CANCELLED;
        }

        gcq_remove_all(&t->to);

        t->flags |= F_UDISABLED;
        t->flags &= ~(F_AREADY|F_BREADY|F_AINPROG|F_BINPROG|F_LOWSPEED);
}

/*
 * RESET: SL11_CTRL_RESETENGINE=1 and SL11_CTRL_JKSTATE=0 for 50ms
 * reconfigure SOF after reset, must wait 2.5us before USB bus activity (SOF)
 * check attached device speed.
 * must wait 100ms before USB transaction according to app note, 10ms
 * by spec.  uhub does this delay
 *
 * Started from root hub set feature reset, which does step one.
 * ub_usepolling will call slhci_reset directly, otherwise the callout goes
 * through slhci_reset_entry.
 */
void
slhci_reset(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        struct slhci_pipe *spipe;
        struct gcq *q;
        uint8_t r, pol, ctrl;

        t = &sc->sc_transfers;
        KASSERT(mutex_owned(&sc->sc_intr_lock));

        stop_cc_time(&t_delay);

        KASSERT(t->flags & F_ACTIVE);

        start_cc_time(&t_delay, 0);
        stop_cc_time(&t_delay);

        slhci_write(sc, SL11_CTRL, 0);
        start_cc_time(&t_delay, 3);
        DELAY(3);
        stop_cc_time(&t_delay);
        slhci_write(sc, SL11_ISR, 0xff);

        r = slhci_read(sc, SL11_ISR);

        if (r & SL11_ISR_INSERT)
                slhci_write(sc, SL11_ISR, SL11_ISR_INSERT);

        if (r & SL11_ISR_NODEV) {
                DLOG(D_MSG, "NC", 0,0,0,0);
                /*
                 * Normally, the hard interrupt insert routine will issue
                 * CCONNECT, however we need to do it here if the detach
                 * happened during reset.
                 */
                if (!(t->flags & F_NODEV))
                        t->flags |= F_CCONNECT|F_ROOTINTR|F_NODEV;
                slhci_intrchange(sc, SL11_IER_INSERT);
        } else {
                if (t->flags & F_NODEV)
                        t->flags |= F_CCONNECT;
                t->flags &= ~(F_NODEV|F_LOWSPEED);
                if (r & SL11_ISR_DATA) {
                        DLOG(D_MSG, "FS", 0,0,0,0);
                        pol = ctrl = 0;
                } else {
                        DLOG(D_MSG, "LS", 0,0,0,0);
                        pol  = SL811_CSOF_POLARITY;
                        ctrl = SL11_CTRL_LOWSPEED;
                        t->flags |= F_LOWSPEED;
                }

                /* Enable SOF auto-generation */
                t->frame = 0;   /* write to SL811_CSOF will reset frame */
                slhci_write(sc, SL11_SOFTIME, 0xe0);
                slhci_write(sc, SL811_CSOF, pol|SL811_CSOF_MASTER|0x2e);
                slhci_write(sc, SL11_CTRL, ctrl|SL11_CTRL_ENABLESOF);

                /*
                 * According to the app note, ARM must be set
                 * for SOF generation to work.  We initialize all
                 * USBA registers here for current_tregs.
                 */
                slhci_write(sc, SL11_E0ADDR, SL11_BUFFER_START);
                slhci_write(sc, SL11_E0LEN, 0);
                slhci_write(sc, SL11_E0PID, SL11_PID_SOF);
                slhci_write(sc, SL11_E0DEV, 0);
                slhci_write(sc, SL11_E0CTRL, SL11_EPCTRL_ARM);

                /*
                 * Initialize B registers.  This can't be done earlier since
                 * they are not valid until the SL811_CSOF register is written
                 * above due to SL11H compatibility.
                 */
                slhci_write(sc, SL11_E1ADDR, SL11_BUFFER_END - 8);
                slhci_write(sc, SL11_E1LEN, 0);
                slhci_write(sc, SL11_E1PID, 0);
                slhci_write(sc, SL11_E1DEV, 0);

                t->current_tregs[0][ADR] = SL11_BUFFER_START;
                t->current_tregs[0][LEN] = 0;
                t->current_tregs[0][PID] = SL11_PID_SOF;
                t->current_tregs[0][DEV] = 0;
                t->current_tregs[1][ADR] = SL11_BUFFER_END - 8;
                t->current_tregs[1][LEN] = 0;
                t->current_tregs[1][PID] = 0;
                t->current_tregs[1][DEV] = 0;

                /* SOF start will produce USBA interrupt */
                t->len[A] = 0;
                t->flags |= F_AINPROG;

                slhci_intrchange(sc, SLHCI_NORMAL_INTERRUPTS);
        }

        t->flags &= ~(F_UDISABLED|F_RESET);
        t->flags |= F_CRESET|F_ROOTINTR;
        FOREACH_AP(q, t, spipe) {
                spipe->pflags &= ~PF_GONE;
                spipe->pipe.up_aborting = 0;
        }
        DLOG(D_MSG, "RESET done flags %#jx", t->flags, 0,0,0);
}


#ifdef SLHCI_DEBUG
static int
slhci_memtest(struct slhci_softc *sc)
{
        enum { ASC, DESC, EITHER = ASC };       /* direction */
        enum { READ, WRITE };                   /* operation */
        const char *ptr, *elem;
        size_t i;
        const int low = SL11_BUFFER_START, high = SL11_BUFFER_END;
        int addr = 0, dir = ASC, op = READ;
        /* Extended March C- test algorithm (SOFs also) */
        const char test[] = "E(w0) A(r0w1r1) A(r1w0r0) D(r0w1) D(r1w0) E(r0)";
        char c;
        const uint8_t dbs[] = { 0x00, 0x0f, 0x33, 0x55 }; /* data backgrounds */
        uint8_t db;

        /* Perform memory test for all data backgrounds. */
        for (i = 0; i < __arraycount(dbs); i++) {
                ptr = test;
                elem = ptr;
                /* Walk test algorithm string. */
                while ((c = *ptr++) != '\0')
                        switch (tolower((int)c)) {
                        case 'a':
                                /* Address sequence is in ascending order. */
                                dir = ASC;
                                break;
                        case 'd':
                                /* Address sequence is in descending order. */
                                dir = DESC;
                                break;
                        case 'e':
                                /* Address sequence is in either order. */
                                dir = EITHER;
                                break;
                        case '(':
                                /* Start of test element (sequence). */
                                elem = ptr;
                                addr = (dir == ASC) ? low : high;
                                break;
                        case 'r':
                                /* read operation */
                                op = READ;
                                break;
                        case 'w':
                                /* write operation */
                                op = WRITE;
                                break;
                        case '0':
                        case '1':
                                /*
                                 * Execute previously set-up operation by
                                 * reading/writing non-inverted ('0') or
                                 * inverted ('1') data background.
                                 */
                                db = (c - '0') ? ~dbs[i] : dbs[i];
                                if (op == READ) {
                                        if (slhci_read(sc, addr) != db)
                                                return -1;
                                } else
                                        slhci_write(sc, addr, db);
                                break;
                        case ')':
                                /*
                                 * End of element: Repeat same element with next
                                 * address or continue to next element.
                                 */
                                addr = (dir == ASC) ? addr + 1 : addr - 1;
                                if (addr >= low && addr <= high)
                                        ptr = elem;
                                break;
                        default:
                                /* Do nothing. */
                                break;
                        }
        }

        return 0;
}
#endif

/* returns 1 if succeeded, 0 if failed, reserve == 0 is unreserve */
static int
slhci_reserve_bustime(struct slhci_softc *sc, struct slhci_pipe *spipe, int
    reserve)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        int bustime, max_packet;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        t = &sc->sc_transfers;
        max_packet = UGETW(spipe->pipe.up_endpoint->ue_edesc->wMaxPacketSize);

        if (spipe->pflags & PF_LS)
                bustime = SLHCI_LS_CONST + SLHCI_LS_DATA_TIME(max_packet);
        else
                bustime = SLHCI_FS_CONST + SLHCI_FS_DATA_TIME(max_packet);

        if (!reserve) {
                t->reserved_bustime -= bustime;
#ifdef DIAGNOSTIC
                if (t->reserved_bustime < 0) {
                        printf("%s: reserved_bustime %d < 0!\n",
                            SC_NAME(sc), t->reserved_bustime);
                        DDOLOG("reserved_bustime %d < 0!",
                            t->reserved_bustime, 0, 0, 0);
                        t->reserved_bustime = 0;
                }
#endif
                return 1;
        }

        if (t->reserved_bustime + bustime > SLHCI_RESERVED_BUSTIME) {
                if (ratecheck(&sc->sc_reserved_warn_rate,
                    &reserved_warn_rate))
#ifdef SLHCI_NO_OVERTIME
                {
                        printf("%s: Max reserved bus time exceeded! "
                            "Erroring request.\n", SC_NAME(sc));
                        DDOLOG("%s: Max reserved bus time exceeded! "
                            "Erroring request.", 0, 0, 0, 0);
                }
                return 0;
#else
                {
                        printf("%s: Reserved bus time exceeds %d!\n",
                            SC_NAME(sc), SLHCI_RESERVED_BUSTIME);
                        DDOLOG("Reserved bus time exceeds %d!",
                            SLHCI_RESERVED_BUSTIME, 0, 0, 0);
                }
#endif
        }

        t->reserved_bustime += bustime;
        return 1;
}

/* Device insertion/removal interrupt */
static void
slhci_insert(struct slhci_softc *sc)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (t->flags & F_NODEV)
                slhci_intrchange(sc, 0);
        else {
                slhci_drain(sc);
                slhci_intrchange(sc, SL11_IER_INSERT);
        }
        t->flags ^= F_NODEV;
        t->flags |= F_ROOTINTR|F_CCONNECT;
        DLOG(D_MSG, "INSERT intr: flags after %#jx", t->flags, 0,0,0);
}

/*
 * Data structures and routines to emulate the root hub.
 */

static usbd_status
slhci_clear_feature(struct slhci_softc *sc, unsigned int what)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        usbd_status error;

        t = &sc->sc_transfers;
        error = USBD_NORMAL_COMPLETION;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (what == UHF_PORT_POWER) {
                DLOG(D_MSG, "POWER_OFF", 0,0,0,0);
                t->flags &= ~F_POWER;
                if (!(t->flags & F_NODEV))
                        t->flags |= F_NODEV|F_CCONNECT|F_ROOTINTR;
                /* for x68k Nereid USB controller */
                if (sc->sc_enable_power && (t->flags & F_REALPOWER)) {
                        t->flags &= ~F_REALPOWER;
                        sc->sc_enable_power(sc, POWER_OFF);
                }
                slhci_intrchange(sc, 0);
                slhci_drain(sc);
        } else if (what == UHF_C_PORT_CONNECTION) {
                t->flags &= ~F_CCONNECT;
        } else if (what == UHF_C_PORT_RESET) {
                t->flags &= ~F_CRESET;
        } else if (what == UHF_PORT_ENABLE) {
                slhci_drain(sc);
        } else if (what != UHF_PORT_SUSPEND) {
                DDOLOG("ClrPortFeatERR:value=%#.4x", what, 0,0,0);
                error = USBD_IOERROR;
        }

        return error;
}

static usbd_status
slhci_set_feature(struct slhci_softc *sc, unsigned int what)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        uint8_t r;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        if (what == UHF_PORT_RESET) {
                if (!(t->flags & F_ACTIVE)) {
                        DDOLOG("SET PORT_RESET when not ACTIVE!",
                            0,0,0,0);
                        return USBD_INVAL;
                }
                if (!(t->flags & F_POWER)) {
                        DDOLOG("SET PORT_RESET without PORT_POWER! flags %p",
                            t->flags, 0,0,0);
                        return USBD_INVAL;
                }
                if (t->flags & F_RESET)
                        return USBD_NORMAL_COMPLETION;
                DLOG(D_MSG, "RESET flags %#jx", t->flags, 0,0,0);
                slhci_intrchange(sc, 0);
                slhci_drain(sc);
                slhci_write(sc, SL11_CTRL, SL11_CTRL_RESETENGINE);
                /* usb spec says delay >= 10ms, app note 50ms */
                start_cc_time(&t_delay, 50000);
                if (sc->sc_bus.ub_usepolling) {
                        DELAY(50000);
                        slhci_reset(sc);
                } else {
                        t->flags |= F_RESET;
                        callout_schedule(&sc->sc_timer, uimax(mstohz(50), 2));
                }
        } else if (what == UHF_PORT_SUSPEND) {
                printf("%s: USB Suspend not implemented!\n", SC_NAME(sc));
                DDOLOG("USB Suspend not implemented!", 0, 0, 0, 0);
        } else if (what == UHF_PORT_POWER) {
                DLOG(D_MSG, "PORT_POWER", 0,0,0,0);
                /* for x68k Nereid USB controller */
                if (!(t->flags & F_ACTIVE))
                        return USBD_INVAL;
                if (t->flags & F_POWER)
                        return USBD_NORMAL_COMPLETION;
                if (!(t->flags & F_REALPOWER)) {
                        if (sc->sc_enable_power)
                                sc->sc_enable_power(sc, POWER_ON);
                        t->flags |= F_REALPOWER;
                }
                t->flags |= F_POWER;
                r = slhci_read(sc, SL11_ISR);
                if (r & SL11_ISR_INSERT)
                        slhci_write(sc, SL11_ISR, SL11_ISR_INSERT);
                if (r & SL11_ISR_NODEV) {
                        slhci_intrchange(sc, SL11_IER_INSERT);
                        t->flags |= F_NODEV;
                } else {
                        t->flags &= ~F_NODEV;
                        t->flags |= F_CCONNECT|F_ROOTINTR;
                }
        } else {
                DDOLOG("SetPortFeatERR=%#.8x", what, 0,0,0);
                return USBD_IOERROR;
        }

        return USBD_NORMAL_COMPLETION;
}

static void
slhci_get_status(struct slhci_softc *sc, usb_port_status_t *ps)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_transfers *t;
        unsigned int status, change;

        t = &sc->sc_transfers;

        KASSERT(mutex_owned(&sc->sc_intr_lock));

        /*
         * We do not have a way to detect over current or babble and
         * suspend is currently not implemented, so connect and reset
         * are the only changes that need to be reported.
         */
        change = 0;
        if (t->flags & F_CCONNECT)
                change |= UPS_C_CONNECT_STATUS;
        if (t->flags & F_CRESET)
                change |= UPS_C_PORT_RESET;

        status = 0;
        if (!(t->flags & F_NODEV))
                status |= UPS_CURRENT_CONNECT_STATUS;
        if (!(t->flags & F_UDISABLED))
                status |= UPS_PORT_ENABLED;
        if (t->flags & F_RESET)
                status |= UPS_RESET;
        if (t->flags & F_POWER)
                status |= UPS_PORT_POWER;
        if (t->flags & F_LOWSPEED)
                status |= UPS_LOW_SPEED;
        USETW(ps->wPortStatus, status);
        USETW(ps->wPortChange, change);
        DLOG(D_ROOT, "status=%#.4jx, change=%#.4jx", status, change, 0,0);
}

static int
slhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req,
    void *buf, int buflen)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        struct slhci_softc *sc = SLHCI_BUS2SC(bus);
        struct slhci_transfers *t = &sc->sc_transfers;
        usbd_status error = USBD_IOERROR; /* XXX should be STALL */
        uint16_t len, value, index;
        uint8_t type;
        int actlen = 0;

        len = UGETW(req->wLength);
        value = UGETW(req->wValue);
        index = UGETW(req->wIndex);

        type = req->bmRequestType;

        SLHCI_DEXEC(D_TRACE, slhci_log_req(req));

        /*
         * USB requests for hubs have two basic types, standard and class.
         * Each could potentially have recipients of device, interface,
         * endpoint, or other.  For the hub class, CLASS_OTHER means the port
         * and CLASS_DEVICE means the hub.  For standard requests, OTHER
         * is not used.  Standard request are described in section 9.4 of the
         * standard, hub class requests in 11.16.  Each request is either read
         * or write.
         *
         * Clear Feature, Set Feature, and Status are defined for each of the
         * used recipients.  Get Descriptor and Set Descriptor are defined for
         * both standard and hub class types with different descriptors.
         * Other requests have only one defined recipient and type.  These
         * include: Get/Set Address, Get/Set Configuration, Get/Set Interface,
         * and Synch Frame for standard requests and Get Bus State for hub
         * class.
         *
         * When a device is first powered up it has address 0 until the
         * address is set.
         *
         * Hubs are only allowed to support one interface and may not have
         * isochronous endpoints.  The results of the related requests are
         * undefined.
         *
         * The standard requires invalid or unsupported requests to return
         * STALL in the data stage, however this does not work well with
         * current error handling. XXX
         *
         * Some unsupported fields:
         * Clear Hub Feature is for C_HUB_LOCAL_POWER and C_HUB_OVER_CURRENT
         * Set Device Features is for ENDPOINT_HALT and DEVICE_REMOTE_WAKEUP
         * Get Bus State is optional sample of D- and D+ at EOF2
         */

        switch (req->bRequest) {
        /* Write Requests */
        case UR_CLEAR_FEATURE:
                if (type == UT_WRITE_CLASS_OTHER) {
                        if (index == 1 /* Port */) {
                                mutex_enter(&sc->sc_intr_lock);
                                error = slhci_clear_feature(sc, value);
                                mutex_exit(&sc->sc_intr_lock);
                        } else
                                DLOG(D_ROOT, "Clear Port Feature "
                                    "index = %#.4jx", index, 0,0,0);
                }
                break;
        case UR_SET_FEATURE:
                if (type == UT_WRITE_CLASS_OTHER) {
                        if (index == 1 /* Port */) {
                                mutex_enter(&sc->sc_intr_lock);
                                error = slhci_set_feature(sc, value);
                                mutex_exit(&sc->sc_intr_lock);
                        } else
                                DLOG(D_ROOT, "Set Port Feature "
                                    "index = %#.4jx", index, 0,0,0);
                } else if (type != UT_WRITE_CLASS_DEVICE)
                        DLOG(D_ROOT, "Set Device Feature "
                            "ENDPOINT_HALT or DEVICE_REMOTE_WAKEUP "
                            "not supported", 0,0,0,0);
                break;

        /* Read Requests */
        case UR_GET_STATUS:
                if (type == UT_READ_CLASS_OTHER) {
                        if (index == 1 /* Port */ && len == /* XXX >=? */
                            sizeof(usb_port_status_t)) {
                                mutex_enter(&sc->sc_intr_lock);
                                slhci_get_status(sc, (usb_port_status_t *)
                                    buf);
                                mutex_exit(&sc->sc_intr_lock);
                                actlen = sizeof(usb_port_status_t);
                                error = USBD_NORMAL_COMPLETION;
                        } else
                                DLOG(D_ROOT, "Get Port Status index = %#.4jx "
                                    "len = %#.4jx", index, len, 0,0);
                } else if (type == UT_READ_CLASS_DEVICE) { /* XXX index? */
                        if (len == sizeof(usb_hub_status_t)) {
                                DLOG(D_ROOT, "Get Hub Status",
                                    0,0,0,0);
                                actlen = sizeof(usb_hub_status_t);
                                memset(buf, 0, actlen);
                                error = USBD_NORMAL_COMPLETION;
                        } else
                                DLOG(D_ROOT, "Get Hub Status bad len %#.4jx",
                                    len, 0,0,0);
                }
                break;
        case UR_GET_DESCRIPTOR:
                if (type == UT_READ_DEVICE) {
                        /* value is type (&0xff00) and index (0xff) */
                        if (value == (UDESC_DEVICE<<8)) {
                                actlen = buflen;
                                error = USBD_NORMAL_COMPLETION;
                        } else if (value == (UDESC_CONFIG<<8)) {
                                struct usb_roothub_descriptors confd;

                                actlen = uimin(buflen, sizeof(confd));
                                memcpy(&confd, buf, actlen);

                                /* 2 mA units */
                                confd.urh_confd.bMaxPower = t->max_current;
                                memcpy(buf, &confd, actlen);
                                error = USBD_NORMAL_COMPLETION;
                        } else if (value == ((UDESC_STRING<<8)|1)) {
                                /* Vendor */
                                actlen = buflen;
                                error = USBD_NORMAL_COMPLETION;
                        } else if (value == ((UDESC_STRING<<8)|2)) {
                                /* Product */
                                actlen = usb_makestrdesc((usb_string_descriptor_t *)
                                    buf, len, "SL811HS/T root hub");
                                error = USBD_NORMAL_COMPLETION;
                        } else
                                DDOLOG("Unknown Get Descriptor %#.4x",
                                    value, 0,0,0);
                } else if (type == UT_READ_CLASS_DEVICE) {
                        /* Descriptor number is 0 */
                        if (value == (UDESC_HUB<<8)) {
                                usb_hub_descriptor_t hubd;

                                actlen = uimin(buflen, sizeof(hubd));
                                memcpy(&hubd, buf, actlen);
                                hubd.bHubContrCurrent =
                                    500 - t->max_current;
                                memcpy(buf, &hubd, actlen);
                                error = USBD_NORMAL_COMPLETION;
                        } else
                                DDOLOG("Unknown Get Hub Descriptor %#.4x",
                                    value, 0,0,0);
                }
                break;
        default:
                /* default from usbroothub */
                return buflen;
        }

        if (error == USBD_NORMAL_COMPLETION)
                return actlen;

        return -1;
}

/* End in lock functions. Start debug functions. */

#ifdef SLHCI_DEBUG
void
slhci_log_buffer(struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        u_char *buf;

        if(xfer->ux_length > 0 &&
            UE_GET_DIR(xfer->ux_pipe->up_endpoint->ue_edesc->bEndpointAddress) ==
            UE_DIR_IN) {
                buf = xfer->ux_buf;
                DDOLOGBUF(buf, xfer->ux_actlen);
                DDOLOG("len %d actlen %d short %d", xfer->ux_length,
                    xfer->ux_actlen, xfer->ux_length - xfer->ux_actlen, 0);
        }
}

void
slhci_log_req(usb_device_request_t *r)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        int req, type, value, index, len;

        req   = r->bRequest;
        type  = r->bmRequestType;
        value = UGETW(r->wValue);
        index = UGETW(r->wIndex);
        len   = UGETW(r->wLength);

        DDOLOG("request: type %#x", type, 0, 0, 0);
        DDOLOG("request: r=%d,v=%d,i=%d,l=%d ", req, value, index, len);
}

void
slhci_log_dumpreg(void)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        uint8_t r;
        unsigned int aaddr, alen, baddr, blen;
        static u_char buf[240];

        r = slhci_read(ssc, SL11_E0CTRL);
        DDOLOG("USB A Host Control = %#.2x", r, 0, 0, 0);
        DDOLOGEPCTRL(r);

        aaddr = slhci_read(ssc, SL11_E0ADDR);
        DDOLOG("USB A Base Address = %u", aaddr, 0,0,0);
        alen = slhci_read(ssc, SL11_E0LEN);
        DDOLOG("USB A Length = %u", alen, 0,0,0);
        r = slhci_read(ssc, SL11_E0STAT);
        DDOLOG("USB A Status = %#.2x", r, 0,0,0);
        DDOLOGEPSTAT(r);

        r = slhci_read(ssc, SL11_E0CONT);
        DDOLOG("USB A Remaining or Overflow Length = %u", r, 0,0,0);
        r = slhci_read(ssc, SL11_E1CTRL);
        DDOLOG("USB B Host Control = %#.2x", r, 0,0,0);
        DDOLOGEPCTRL(r);

        baddr = slhci_read(ssc, SL11_E1ADDR);
        DDOLOG("USB B Base Address = %u", baddr, 0,0,0);
        blen = slhci_read(ssc, SL11_E1LEN);
        DDOLOG("USB B Length = %u", blen, 0,0,0);
        r = slhci_read(ssc, SL11_E1STAT);
        DDOLOG("USB B Status = %#.2x", r, 0,0,0);
        DDOLOGEPSTAT(r);

        r = slhci_read(ssc, SL11_E1CONT);
        DDOLOG("USB B Remaining or Overflow Length = %u", r, 0,0,0);

        r = slhci_read(ssc, SL11_CTRL);
        DDOLOG("Control = %#.2x", r, 0,0,0);
        DDOLOGCTRL(r);

        r = slhci_read(ssc, SL11_IER);
        DDOLOG("Interrupt Enable = %#.2x", r, 0,0,0);
        DDOLOGIER(r);

        r = slhci_read(ssc, SL11_ISR);
        DDOLOG("Interrupt Status = %#.2x", r, 0,0,0);
        DDOLOGISR(r);

        r = slhci_read(ssc, SL11_REV);
        DDOLOG("Revision = %#.2x", r, 0,0,0);
        r = slhci_read(ssc, SL811_CSOF);
        DDOLOG("SOF Counter = %#.2x", r, 0,0,0);

        if (alen && aaddr >= SL11_BUFFER_START && aaddr < SL11_BUFFER_END &&
            alen <= SL11_MAX_PACKET_SIZE && aaddr + alen <= SL11_BUFFER_END) {
                slhci_read_multi(ssc, aaddr, buf, alen);
                DDOLOG("USBA Buffer: start %u len %u", aaddr, alen, 0,0);
                DDOLOGBUF(buf, alen);
        } else if (alen)
                DDOLOG("USBA Buffer Invalid", 0,0,0,0);

        if (blen && baddr >= SL11_BUFFER_START && baddr < SL11_BUFFER_END &&
            blen <= SL11_MAX_PACKET_SIZE && baddr + blen <= SL11_BUFFER_END) {
                slhci_read_multi(ssc, baddr, buf, blen);
                DDOLOG("USBB Buffer: start %u len %u", baddr, blen, 0,0);
                DDOLOGBUF(buf, blen);
        } else if (blen)
                DDOLOG("USBB Buffer Invalid", 0,0,0,0);
}

void
slhci_log_xfer(struct usbd_xfer *xfer)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        DDOLOG("xfer: length=%u, actlen=%u, flags=%#x, timeout=%u,",
                xfer->ux_length, xfer->ux_actlen, xfer->ux_flags, xfer->ux_timeout);
        DDOLOG("buffer=%p", xfer->ux_buf, 0,0,0);
        slhci_log_req(&xfer->ux_request);
}

void
slhci_log_spipe(struct slhci_pipe *spipe)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        DDOLOG("spipe %p onlists: AP=%d TO=%d XQ=%d", spipe,
            gcq_onlist(&spipe->ap) ? 1 : 0,
            gcq_onlist(&spipe->to) ? 1 : 0,
            gcq_onlist(&spipe->xq) ? 1 : 0);
        DDOLOG("spipe: xfer %p buffer %p pflags %#x ptype %d",
            spipe->xfer, spipe->buffer, spipe->pflags, spipe->ptype);
}

void
slhci_print_intr(void)
{
        unsigned int ier, isr;
        ier = slhci_read(ssc, SL11_IER);
        isr = slhci_read(ssc, SL11_ISR);
        printf("IER: %#x ISR: %#x \n", ier, isr);
}

#if 0
void
slhci_log_sc(void)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();

        struct slhci_transfers *t;
        int i;

        t = &ssc->sc_transfers;

        DDOLOG("Flags=%#x", t->flags, 0,0,0);
        DDOLOG("a = %p Alen=%d b = %p Blen=%d", t->spipe[0], t->len[0],
            t->spipe[1], t->len[1]);

        for (i = 0; i <= Q_MAX; i++)
                DDOLOG("Q %d: %p", i, gcq_hq(&t->q[i]), 0,0);

        DDOLOG("TIMED: %p", GCQ_ITEM(gcq_hq(&t->to),
            struct slhci_pipe, to), 0,0,0);

        DDOLOG("frame=%d rootintr=%p", t->frame, t->rootintr, 0,0);

        DDOLOG("ub_usepolling=%d", ssc->sc_bus.ub_usepolling, 0, 0, 0);
}

void
slhci_log_slreq(struct slhci_pipe *r)
{
        SLHCIHIST_FUNC(); SLHCIHIST_CALLED();
        DDOLOG("xfer: %p", r->xfer, 0,0,0);
        DDOLOG("buffer: %p", r->buffer, 0,0,0);
        DDOLOG("bustime: %u", r->bustime, 0,0,0);
        DDOLOG("control: %#x", r->control, 0,0,0);
        DDOLOGEPCTRL(r->control);

        DDOLOG("pid: %#x", r->tregs[PID], 0,0,0);
        DDOLOG("dev: %u", r->tregs[DEV], 0,0,0);
        DDOLOG("len: %u", r->tregs[LEN], 0,0,0);

        if (r->xfer)
                slhci_log_xfer(r->xfer);
}
#endif
#endif /* SLHCI_DEBUG */
/* End debug functions. */