#include <sys/param.h>
#include <sys/systm.h>
#include <sys/buf.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/device.h>
#include <sys/queue.h>
#include <sys/mutex.h>
#include <sys/pool.h>
#include <machine/bus.h>
#include <dev/ic/ahcireg.h>
#include <dev/ic/ahcivar.h>
#ifdef AHCI_DEBUG
#define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
while (0)
#define AHCI_D_TIMEOUT 0x00
#define AHCI_D_VERBOSE 0x01
#define AHCI_D_INTR 0x02
#define AHCI_D_XFER 0x08
int ahcidebug = AHCI_D_VERBOSE;
#else
#define DPRINTF(m, f...)
#endif
#ifdef HIBERNATE
#include <uvm/uvm_extern.h>
#include <sys/hibernate.h>
#include <sys/disk.h>
#include <sys/disklabel.h>
#include <scsi/scsi_all.h>
#include <scsi/scsiconf.h>
void ahci_hibernate_io_start(struct ahci_port *,
struct ahci_ccb *);
int ahci_hibernate_io_poll(struct ahci_port *,
struct ahci_ccb *);
void ahci_hibernate_load_prdt(struct ahci_ccb *);
int ahci_hibernate_io(dev_t dev, daddr_t blkno,
vaddr_t addr, size_t size, int wr, void *page);
#endif
struct cfdriver ahci_cd = {
NULL, "ahci", DV_DULL
};
void ahci_enable_interrupts(struct ahci_port *);
int ahci_init(struct ahci_softc *);
int ahci_port_alloc(struct ahci_softc *, u_int);
void ahci_port_detect(struct ahci_softc *, u_int);
void ahci_port_free(struct ahci_softc *, u_int);
int ahci_port_init(struct ahci_softc *, u_int);
int ahci_default_port_start(struct ahci_port *, int);
int ahci_port_stop(struct ahci_port *, int);
int ahci_port_clo(struct ahci_port *);
int ahci_port_softreset(struct ahci_port *);
void ahci_port_comreset(struct ahci_port *);
int ahci_port_portreset(struct ahci_port *, int);
void ahci_port_portreset_start(struct ahci_port *);
int ahci_port_portreset_poll(struct ahci_port *);
void ahci_port_portreset_wait(struct ahci_port *);
int ahci_port_portreset_finish(struct ahci_port *, int);
int ahci_port_signature(struct ahci_port *);
int ahci_pmp_port_softreset(struct ahci_port *, int);
int ahci_pmp_port_portreset(struct ahci_port *, int);
int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
int ahci_load_prdt(struct ahci_ccb *);
void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
u_int32_t, u_int32_t);
void ahci_unload_prdt(struct ahci_ccb *);
int ahci_poll(struct ahci_ccb *, int, void (*)(void *));
void ahci_start(struct ahci_ccb *);
void ahci_issue_pending_ncq_commands(struct ahci_port *);
void ahci_issue_pending_commands(struct ahci_port *, int);
int ahci_intr(void *);
u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
struct ahci_ccb *ahci_get_ccb(struct ahci_port *);
void ahci_put_ccb(struct ahci_ccb *);
struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
void ahci_put_err_ccb(struct ahci_ccb *);
struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *);
void ahci_put_pmp_ccb(struct ahci_ccb *);
int ahci_port_read_ncq_error(struct ahci_port *, int *, int);
struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t);
void ahci_dmamem_free(struct ahci_softc *,
struct ahci_dmamem *);
u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
int ahci_wait_ne(struct ahci_softc *, bus_size_t,
u_int32_t, u_int32_t);
u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
int ahci_pwait_eq(struct ahci_port *, bus_size_t,
u_int32_t, u_int32_t, int);
void ahci_flush_tfd(struct ahci_port *ap);
u_int32_t ahci_active_mask(struct ahci_port *);
int ahci_port_detect_pmp(struct ahci_port *);
void ahci_pmp_probe_timeout(void *);
int ahci_pmp_read(struct ahci_port *, int, int,
u_int32_t *);
int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
int ahci_pmp_phy_status(struct ahci_port *, int,
u_int32_t *);
int ahci_pmp_identify(struct ahci_port *, int *);
#define ahci_pwait_clr(_ap, _r, _b, _n) \
ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
#define ahci_pwait_set(_ap, _r, _b, _n) \
ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
int ahci_ata_probe(void *, int, int);
void ahci_ata_free(void *, int, int);
struct ata_xfer * ahci_ata_get_xfer(void *, int);
void ahci_ata_put_xfer(struct ata_xfer *);
void ahci_ata_cmd(struct ata_xfer *);
const struct atascsi_methods ahci_atascsi_methods = {
ahci_ata_probe,
ahci_ata_free,
ahci_ata_get_xfer,
ahci_ata_put_xfer,
ahci_ata_cmd
};
void ahci_ata_cmd_done(struct ahci_ccb *);
void ahci_pmp_cmd_done(struct ahci_ccb *);
void ahci_ata_cmd_timeout(void *);
void ahci_empty_done(struct ahci_ccb *);
int
ahci_attach(struct ahci_softc *sc)
{
struct atascsi_attach_args aaa;
u_int32_t pi;
int i, j, done;
if (sc->sc_port_start == NULL)
sc->sc_port_start = ahci_default_port_start;
if (ahci_init(sc) != 0) {
goto unmap;
}
printf("\n");
sc->sc_cap = ahci_read(sc, AHCI_REG_CAP);
sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap);
#ifdef AHCI_DEBUG
if (ahcidebug & AHCI_D_VERBOSE) {
const char *gen;
switch (sc->sc_cap & AHCI_REG_CAP_ISS) {
case AHCI_REG_CAP_ISS_G1:
gen = "1 (1.5Gbps)";
break;
case AHCI_REG_CAP_ISS_G2:
gen = "2 (3.0Gb/s)";
break;
case AHCI_REG_CAP_ISS_G3:
gen = "3 (6.0Gb/s)";
break;
default:
gen = "unknown";
break;
}
printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP,
AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen);
printf("%s: extended capabilities 0x%b\n", DEVNAME(sc),
ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2);
}
#endif
pi = ahci_read(sc, AHCI_REG_PI);
DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
DEVNAME(sc), pi);
#ifdef AHCI_COALESCE
if (sc->sc_cap & AHCI_REG_CAP_CCCS) {
u_int16_t ccc_timeout = 20;
u_int8_t ccc_numcomplete = 12;
u_int32_t ccc_ctl;
ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
ccc_ctl &= ~0x00000001;
ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
if (pi & sc->sc_ccc_mask) {
printf("%s: coalescing interrupt/implemented port list "
"conflict, PI: %08x, ccc_mask: %08x\n",
DEVNAME(sc), pi, sc->sc_ccc_mask);
sc->sc_ccc_mask = 0;
goto noccc;
}
sc->sc_ccc_ports = pi;
sc->sc_ccc_ports_cur = 0;
ccc_ctl &= ~0xffffff00;
ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
}
noccc:
#endif
sc->sc_ncmds = max(2, sc->sc_ncmds);
for (i = 0; i < AHCI_MAX_PORTS; i++) {
if (!ISSET(pi, 1U << i)) {
continue;
}
if (ahci_port_alloc(sc, i) == ENOMEM)
goto freeports;
if (sc->sc_ports[i] != NULL)
ahci_port_portreset_start(sc->sc_ports[i]);
}
for (i = 0; i < 1000; i++) {
done = 1;
for (j = 0; j < AHCI_MAX_PORTS; j++) {
if (sc->sc_ports[j] == NULL)
continue;
if (ahci_port_portreset_poll(sc->sc_ports[j]))
done = 0;
}
if (done)
break;
delay(1000);
}
for (i = 0; i < AHCI_MAX_PORTS; i++) {
if (sc->sc_ports[i] != NULL)
ahci_port_detect(sc, i);
}
memset(&aaa, 0, sizeof(aaa));
aaa.aaa_cookie = sc;
aaa.aaa_methods = &ahci_atascsi_methods;
aaa.aaa_minphys = NULL;
aaa.aaa_nports = AHCI_MAX_PORTS;
aaa.aaa_ncmds = sc->sc_ncmds - 1;
if (!(sc->sc_flags & AHCI_F_NO_NCQ) &&
sc->sc_ncmds > 2 &&
(sc->sc_cap & AHCI_REG_CAP_SNCQ)) {
aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ;
}
sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
ahci_write(sc, AHCI_REG_IS, ahci_read(sc, AHCI_REG_IS));
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
return 0;
freeports:
for (i = 0; i < AHCI_MAX_PORTS; i++) {
if (sc->sc_ports[i] != NULL)
ahci_port_free(sc, i);
}
unmap:
ahci_write(sc, AHCI_REG_GHC, 0);
return 1;
}
int
ahci_detach(struct ahci_softc *sc, int flags)
{
int rv, i;
if (sc->sc_atascsi != NULL) {
rv = atascsi_detach(sc->sc_atascsi, flags);
if (rv != 0)
return (rv);
}
for (i = 0; i < AHCI_MAX_PORTS; i++) {
if (sc->sc_ports[i] != NULL)
ahci_port_free(sc, i);
}
return (0);
}
int
ahci_activate(struct device *self, int act)
{
struct ahci_softc *sc = (struct ahci_softc *)self;
int i, rv = 0;
switch (act) {
case DVACT_RESUME:
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
ahci_write(sc, AHCI_REG_CAP, sc->sc_cap);
for (i = 0; i < AHCI_MAX_PORTS; i++) {
if (sc->sc_ports[i] != NULL)
ahci_port_init(sc, i);
}
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
rv = config_activate_children(self, act);
break;
case DVACT_POWERDOWN:
rv = config_activate_children(self, act);
for (i = 0; i < AHCI_MAX_PORTS; i++) {
if (sc->sc_ports[i] != NULL)
ahci_port_stop(sc->sc_ports[i], 1);
}
break;
default:
rv = config_activate_children(self, act);
break;
}
return (rv);
}
int
ahci_init(struct ahci_softc *sc)
{
u_int32_t reg, cap, pi;
const char *revision;
DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
AHCI_FMT_GHC);
cap = ahci_read(sc, AHCI_REG_CAP);
cap &= AHCI_REG_CAP_SMPS;
cap |= AHCI_REG_CAP_SSS;
pi = ahci_read(sc, AHCI_REG_PI);
if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
AHCI_REG_GHC_HR) != 0) {
printf(" unable to reset controller\n");
return (1);
}
}
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
ahci_write(sc, AHCI_REG_CAP, cap);
ahci_write(sc, AHCI_REG_PI, pi);
reg = ahci_read(sc, AHCI_REG_VS);
switch (reg) {
case AHCI_REG_VS_0_95:
revision = "0.95";
break;
case AHCI_REG_VS_1_0:
revision = "1.0";
break;
case AHCI_REG_VS_1_1:
revision = "1.1";
break;
case AHCI_REG_VS_1_2:
revision = "1.2";
break;
case AHCI_REG_VS_1_3:
revision = "1.3";
break;
case AHCI_REG_VS_1_3_1:
revision = "1.3.1";
break;
default:
printf(" unsupported AHCI revision 0x%08x\n", reg);
return (1);
}
printf(" AHCI %s", revision);
return (0);
}
void
ahci_enable_interrupts(struct ahci_port *ap)
{
ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
AHCI_PREG_IE_UFE |
((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) |
#ifdef AHCI_COALESCE
((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
(AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE))
#else
AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
#endif
);
}
int
ahci_port_alloc(struct ahci_softc *sc, u_int port)
{
struct ahci_port *ap;
struct ahci_ccb *ccb;
u_int64_t dva;
u_int32_t cmd;
struct ahci_cmd_hdr *hdr;
struct ahci_cmd_table *table;
int i, rc = ENOMEM;
ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO);
if (ap == NULL) {
printf("%s: unable to allocate memory for port %d\n",
DEVNAME(sc), port);
goto reterr;
}
ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO);
if (ap->ap_err_scratch == NULL) {
printf("%s: unable to allocate DMA scratch buf for port %d\n",
DEVNAME(sc), port);
free(ap, M_DEVBUF, sizeof(*ap));
goto reterr;
}
#ifdef AHCI_DEBUG
snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
DEVNAME(sc), port);
#endif
ap->ap_port = port;
sc->sc_ports[port] = ap;
if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
printf("%s: unable to create register window for port %d\n",
DEVNAME(sc), port);
goto freeport;
}
ap->ap_sc = sc;
#ifdef AHCI_COALESCE
ap->ap_num = port;
#endif
TAILQ_INIT(&ap->ap_ccb_free);
TAILQ_INIT(&ap->ap_ccb_pending);
mtx_init(&ap->ap_ccb_mtx, IPL_BIO);
ahci_pwrite(ap, AHCI_PREG_IE, 0);
cmd = ahci_pread(ap, AHCI_PREG_CMD);
if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
int r;
r = ahci_port_stop(ap, 1);
if (r) {
printf("%s: unable to disable %s, ignoring port %d\n",
DEVNAME(sc), r == 2 ? "CR" : "FR", port);
rc = ENXIO;
goto freeport;
}
ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
}
ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
if (ap->ap_dmamem_rfis == NULL)
goto nomem;
ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
rc = ENXIO;
goto freeport;
}
ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
M_DEVBUF, M_NOWAIT | M_ZERO);
if (ap->ap_ccbs == NULL) {
printf("%s: unable to allocate command list for port %d\n",
DEVNAME(sc), port);
goto freeport;
}
ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
sc->sc_ncmds * sizeof(struct ahci_cmd_table));
if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
nomem:
printf("%s: unable to allocate DMA memory for port %d\n",
DEVNAME(sc), port);
goto freeport;
}
dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
for (i = 0; i < sc->sc_ncmds; i++) {
ccb = &ap->ap_ccbs[i];
if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
(4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
&ccb->ccb_dmamap) != 0) {
printf("%s: unable to create dmamap for port %d "
"ccb %d\n", DEVNAME(sc), port, i);
goto freeport;
}
ccb->ccb_slot = i;
ccb->ccb_port = ap;
ccb->ccb_cmd_hdr = &hdr[i];
ccb->ccb_cmd_table = &table[i];
htolem64(&ccb->ccb_cmd_hdr->ctba,
AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
ccb->ccb_slot * sizeof(struct ahci_cmd_table));
ccb->ccb_xa.fis =
(struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
ccb->ccb_xa.tag = i;
ccb->ccb_xa.state = ATA_S_COMPLETE;
ahci_put_ccb(ccb);
}
ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry);
ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE;
ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
rc = 0;
freeport:
if (rc != 0)
ahci_port_free(sc, port);
reterr:
return (rc);
}
void
ahci_port_detect(struct ahci_softc *sc, u_int port)
{
struct ahci_port *ap;
const char *speed;
int rc;
ap = sc->sc_ports[port];
rc = ahci_port_portreset_finish(ap, 1);
switch (rc) {
case ENODEV:
switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
case AHCI_PREG_SSTS_DET_DEV_NE:
printf("%s: device not communicating on port %d\n",
DEVNAME(sc), port);
break;
case AHCI_PREG_SSTS_DET_PHYOFFLINE:
printf("%s: PHY offline on port %d\n", DEVNAME(sc),
port);
break;
default:
DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
"on port %d\n", DEVNAME(sc), port);
break;
}
goto freeport;
case EBUSY:
printf("%s: device on port %d didn't come ready, "
"TFD: 0x%b\n", DEVNAME(sc), port,
ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
rc = ahci_port_softreset(ap);
if (rc) {
printf("%s: unable to communicate "
"with device on port %d\n", DEVNAME(sc), port);
goto freeport;
}
break;
default:
break;
}
DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
DEVNAME(sc), port, rc);
switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) {
case AHCI_PREG_SSTS_SPD_GEN1:
speed = "1.5Gb/s";
break;
case AHCI_PREG_SSTS_SPD_GEN2:
speed = "3.0Gb/s";
break;
case AHCI_PREG_SSTS_SPD_GEN3:
speed = "6.0Gb/s";
break;
default:
speed = NULL;
break;
}
if (speed != NULL)
printf("%s: port %d: %s\n", PORTNAME(ap), port, speed);
if (ahci_port_start(ap, 0)) {
printf("%s: failed to start command DMA on port %d, "
"disabling\n", DEVNAME(sc), port);
rc = ENXIO;
}
ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
ahci_write(sc, AHCI_REG_IS, 1 << port);
ahci_enable_interrupts(ap);
freeport:
if (rc != 0)
ahci_port_free(sc, port);
}
void
ahci_port_free(struct ahci_softc *sc, u_int port)
{
struct ahci_port *ap = sc->sc_ports[port];
struct ahci_ccb *ccb;
if (ap->ap_sc) {
ahci_pwrite(ap, AHCI_PREG_CMD, 0);
ahci_pwrite(ap, AHCI_PREG_IE, 0);
ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
ahci_write(sc, AHCI_REG_IS, 1 << port);
}
if (ap->ap_ccb_err)
ahci_put_ccb(ap->ap_ccb_err);
if (ap->ap_ccbs) {
while ((ccb = ahci_get_ccb(ap)) != NULL)
bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb));
}
if (ap->ap_dmamem_cmd_list)
ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
if (ap->ap_dmamem_rfis)
ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
if (ap->ap_dmamem_cmd_table)
ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
if (ap->ap_err_scratch)
dma_free(ap->ap_err_scratch, DEV_BSIZE);
free(ap, M_DEVBUF, sizeof(*ap));
sc->sc_ports[port] = NULL;
}
int
ahci_port_init(struct ahci_softc *sc, u_int port)
{
struct ahci_port *ap;
u_int64_t dva;
u_int32_t cmd;
int rc = ENOMEM;
ap = sc->sc_ports[port];
#ifdef AHCI_DEBUG
snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
DEVNAME(sc), port);
#endif
ahci_pwrite(ap, AHCI_PREG_IE, 0);
cmd = ahci_pread(ap, AHCI_PREG_CMD);
if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
int r;
r = ahci_port_stop(ap, 1);
if (r) {
printf("%s: unable to disable %s, ignoring port %d\n",
DEVNAME(sc), r == 2 ? "CR" : "FR", port);
rc = ENXIO;
goto reterr;
}
ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
}
ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
rc = ENXIO;
goto reterr;
}
dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
rc = ahci_port_portreset(ap, 1);
switch (rc) {
case ENODEV:
switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
case AHCI_PREG_SSTS_DET_DEV_NE:
printf("%s: device not communicating on port %d\n",
DEVNAME(sc), port);
break;
case AHCI_PREG_SSTS_DET_PHYOFFLINE:
printf("%s: PHY offline on port %d\n", DEVNAME(sc),
port);
break;
default:
DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
"on port %d\n", DEVNAME(sc), port);
break;
}
goto reterr;
case EBUSY:
printf("%s: device on port %d didn't come ready, "
"TFD: 0x%b\n", DEVNAME(sc), port,
ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
rc = ahci_port_softreset(ap);
if (rc) {
printf("%s: unable to communicate "
"with device on port %d\n", DEVNAME(sc), port);
goto reterr;
}
break;
default:
break;
}
DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
DEVNAME(sc), port);
if (ap->ap_pmp_ports > 0) {
int p;
for (p = 0; p < ap->ap_pmp_ports; p++) {
int sig;
if (ahci_pmp_port_softreset(ap, p)) {
printf("%s.%d: unable to probe PMP port due to"
" softreset failure\n", PORTNAME(ap), p);
continue;
}
sig = ahci_port_signature(ap);
printf("%s.%d: port signature returned %d\n",
PORTNAME(ap), p, sig);
}
}
if (ahci_port_start(ap, 0)) {
printf("%s: failed to start command DMA on port %d, "
"disabling\n", DEVNAME(sc), port);
rc = ENXIO;
}
ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
ahci_write(sc, AHCI_REG_IS, 1 << port);
ahci_enable_interrupts(ap);
reterr:
return (rc);
}
int
ahci_default_port_start(struct ahci_port *ap, int fre_only)
{
u_int32_t r;
r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
r |= AHCI_PREG_CMD_FRE;
if (!fre_only)
r |= AHCI_PREG_CMD_ST;
ahci_pwrite(ap, AHCI_PREG_CMD, r);
#ifdef AHCI_COALESCE
if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
ap->ap_sc->sc_ccc_ports_cur);
}
#endif
return (0);
}
int
ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
{
u_int32_t r;
#ifdef AHCI_COALESCE
if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
ap->ap_sc->sc_ccc_ports_cur);
}
#endif
r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
r &= ~AHCI_PREG_CMD_ST;
if (stop_fis_rx)
r &= ~AHCI_PREG_CMD_FRE;
ahci_pwrite(ap, AHCI_PREG_CMD, r);
if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1))
return (1);
if (stop_fis_rx &&
ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1))
return (2);
return (0);
}
int
ahci_port_clo(struct ahci_port *ap)
{
struct ahci_softc *sc = ap->ap_sc;
u_int32_t cmd;
if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
return (1);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
#ifdef DIAGNOSTIC
if (ISSET(cmd, AHCI_PREG_CMD_ST))
printf("%s: CLO requested while port running\n", PORTNAME(ap));
#endif
ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) {
printf("%s: CLO did not complete\n", PORTNAME(ap));
return (1);
}
return (0);
}
int
ahci_port_softreset(struct ahci_port *ap)
{
struct ahci_ccb *ccb = NULL;
struct ahci_cmd_hdr *cmd_slot;
u_int8_t *fis;
int s, rc = EIO, oldstate;
u_int32_t cmd;
DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
s = splbio();
oldstate = ap->ap_state;
ap->ap_state = AP_S_ERROR_RECOVERY;
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
if (ahci_port_stop(ap, 0)) {
printf("%s: failed to stop port, cannot softreset\n",
PORTNAME(ap));
goto err;
}
if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
AHCI_PREG_TFD_STS_DRQ))
ahci_port_clo(ap);
ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
if (ahci_port_start(ap, 0)) {
printf("%s: failed to start port, cannot softreset\n",
PORTNAME(ap));
goto err;
}
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
? "failed" : "unsupported");
rc = EBUSY;
goto err;
}
ccb = ahci_get_err_ccb(ap);
cmd_slot = ccb->ccb_cmd_hdr;
memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
fis = ccb->ccb_cmd_table->cfis;
fis[0] = ATA_FIS_TYPE_H2D;
fis[15] = ATA_FIS_CONTROL_SRST;
cmd_slot->prdtl = 0;
htolem16(&cmd_slot->flags, 5 |
AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
AHCI_CMD_LIST_FLAG_W);
ccb->ccb_xa.state = ATA_S_PENDING;
if (ahci_poll(ccb, 1000, NULL) != 0)
goto err;
fis[0] = ATA_FIS_TYPE_H2D;
fis[15] = 0;
cmd_slot->prdtl = 0;
htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W);
ccb->ccb_xa.state = ATA_S_PENDING;
if (ahci_poll(ccb, 1000, NULL) != 0)
goto err;
if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) {
printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
AHCI_PFMT_TFD_STS);
rc = EBUSY;
goto err;
}
rc = 0;
err:
if (ccb != NULL) {
if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
printf("%s: stopping the port, softreset slot %d was "
"still active.\n", PORTNAME(ap), ccb->ccb_slot);
ahci_port_stop(ap, 0);
}
ccb->ccb_xa.state = ATA_S_ERROR;
ahci_put_err_ccb(ccb);
}
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
ap->ap_state = oldstate;
splx(s);
return (rc);
}
int
ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
{
struct ahci_ccb *ccb = NULL;
u_int32_t data;
int count;
int rc;
int s;
struct ahci_cmd_hdr *cmd_slot;
u_int8_t *fis;
s = splbio();
DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
ap->ap_pmp_ignore_ifs = 1;
count = 2;
rc = 0;
do {
if (ccb != NULL) {
ahci_put_pmp_ccb(ccb);
ccb = NULL;
}
if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
printf("%s.%d: unable to clear PHY status\n",
PORTNAME(ap), pmp_port);
}
ahci_pwrite(ap, AHCI_PREG_SERR, -1);
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
ccb = ahci_get_pmp_ccb(ap);
cmd_slot = ccb->ccb_cmd_hdr;
memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
fis = ccb->ccb_cmd_table->cfis;
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = pmp_port;
fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
htolem16(&cmd_slot->flags, 5 |
AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
ccb->ccb_xa.state = ATA_S_PENDING;
DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
PORTNAME(ap), pmp_port);
if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
printf("%s.%d: PMP port softreset cmd failed\n",
PORTNAME(ap), pmp_port);
rc = EBUSY;
if (count > 0) {
}
continue;
}
memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = pmp_port;
fis[15] = ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
htolem16(&cmd_slot->flags, 5 |
(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
PORTNAME(ap), pmp_port);
ccb->ccb_xa.state = ATA_S_PENDING;
if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
"failed\n", PORTNAME(ap), pmp_port);
rc = EBUSY;
if (count > 0) {
}
continue;
}
fis[15] = 0;
break;
} while (count--);
if (ccb != NULL) {
ahci_put_pmp_ccb(ccb);
ccb = NULL;
}
ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
ahci_pwrite(ap, AHCI_PREG_SERR, -1);
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
ap->ap_pmp_ignore_ifs = 0;
DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
splx(s);
return (rc);
}
int
ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
{
int sig;
ap->ap_state = AP_S_PMP_PORT_PROBE;
DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
pmp_port);
if (ahci_pmp_port_portreset(ap, pmp_port)) {
printf("%s.%d: unable to probe PMP port; portreset failed\n",
PORTNAME(ap), pmp_port);
ap->ap_state = AP_S_NORMAL;
return (ATA_PORT_T_NONE);
}
if (ahci_pmp_port_softreset(ap, pmp_port)) {
printf("%s.%d: unable to probe PMP port due to softreset "
"failure\n", PORTNAME(ap), pmp_port);
ap->ap_state = AP_S_NORMAL;
return (ATA_PORT_T_NONE);
}
sig = ahci_port_signature(ap);
DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
PORTNAME(ap), pmp_port, sig);
ap->ap_state = AP_S_NORMAL;
return (sig);
}
void
ahci_flush_tfd(struct ahci_port *ap)
{
u_int32_t r;
r = ahci_pread(ap, AHCI_PREG_SERR);
if (r & AHCI_PREG_SERR_DIAG_X)
ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X);
}
u_int32_t
ahci_active_mask(struct ahci_port *ap)
{
u_int32_t mask;
mask = ahci_pread(ap, AHCI_PREG_CI);
if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ)
mask |= ahci_pread(ap, AHCI_PREG_SACT);
return mask;
}
void
ahci_pmp_probe_timeout(void *cookie)
{
struct ahci_ccb *ccb = cookie;
struct ahci_port *ap = ccb->ccb_port;
u_int32_t mask;
DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
switch (ccb->ccb_xa.state) {
case ATA_S_PENDING:
TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
ccb->ccb_xa.state = ATA_S_TIMEOUT;
break;
case ATA_S_ONCHIP:
case ATA_S_ERROR:
KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&
ap->ap_sactive == 0);
ahci_port_stop(ap, 0);
ahci_port_start(ap, 0);
if (ahci_active_mask(ap) != 0) {
ahci_port_stop(ap, 0);
ahci_port_start(ap, 0);
mask = ahci_active_mask(ap);
if (mask != 0) {
printf("%s: ahci_pmp_probe_timeout: failed to "
"clear active cmds: %08x\n", PORTNAME(ap),
mask);
}
}
ccb->ccb_xa.state = ATA_S_TIMEOUT;
ap->ap_active &= ~(1 << ccb->ccb_slot);
KASSERT(ap->ap_active_cnt > 0);
--ap->ap_active_cnt;
DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, "
"active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot,
ap->ap_active, ap->ap_active_cnt);
break;
default:
panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
PORTNAME(ap), ccb->ccb_xa.state);
}
}
int
ahci_port_signature(struct ahci_port *ap)
{
u_int32_t sig;
sig = ahci_pread(ap, AHCI_PREG_SIG);
if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
return (ATA_PORT_T_ATAPI);
else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER &
0xffff0000))
return (ATA_PORT_T_PM);
else
return (ATA_PORT_T_DISK);
}
int
ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
{
u_int32_t cmd, data;
int loop;
int rc = 1;
int s;
s = splbio();
DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
pmp_port);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
data = AHCI_PREG_SCTL_IPM_DISABLED;
if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1))
goto err;
if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
goto err;
delay(10000);
data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
pmp_port);
data |= AHCI_PREG_SCTL_SPD_GEN1;
} else
data |= AHCI_PREG_SCTL_SPD_ANY;
if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
goto err;
delay(100000);
if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap),
pmp_port);
}
ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE;
if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
goto err;
for (loop = 3; loop; --loop) {
if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
goto err;
if (data & AHCI_PREG_SSTS_DET)
break;
delay(100000);
}
if (loop == 0) {
printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port);
goto err;
}
for (loop = 30; loop; --loop) {
if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
goto err;
if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV)
break;
delay(100000);
}
if (loop == 0) {
printf("%s.%d: device is not negotiating\n", PORTNAME(ap),
pmp_port);
goto err;
}
DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
pmp_port);
delay(100000);
ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
ahci_pwrite(ap, AHCI_PREG_SERR, -1);
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
rc = 0;
err:
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
splx(s);
return (rc);
}
void
ahci_port_comreset(struct ahci_port *ap)
{
u_int32_t r;
r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
r |= AHCI_PREG_SCTL_SPD_GEN1;
} else
r |= AHCI_PREG_SCTL_SPD_ANY;
ahci_pwrite(ap, AHCI_PREG_SCTL, r);
delay(10000);
r &= ~AHCI_PREG_SCTL_DET_INIT;
r |= AHCI_PREG_SCTL_DET_NONE;
ahci_pwrite(ap, AHCI_PREG_SCTL, r);
delay(10000);
}
void
ahci_port_portreset_start(struct ahci_port *ap)
{
int s;
s = splbio();
DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
ahci_port_stop(ap, 0);
ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
delay(10000);
ahci_port_comreset(ap);
splx(s);
}
int
ahci_port_portreset_poll(struct ahci_port *ap)
{
if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) !=
AHCI_PREG_SSTS_DET_DEV)
return (EAGAIN);
return (0);
}
void
ahci_port_portreset_wait(struct ahci_port *ap)
{
int i;
for (i = 0; i < 1000; i++) {
if (ahci_port_portreset_poll(ap) == 0)
break;
delay(1000);
}
}
int
ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
{
int rc, s, retries = 0;
s = splbio();
retry:
if (ahci_port_portreset_poll(ap)) {
rc = ENODEV;
if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
} else {
goto err;
}
} else {
ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) {
if (retries == 0) {
retries = 1;
ahci_port_comreset(ap);
ahci_port_portreset_wait(ap);
goto retry;
}
rc = EBUSY;
} else {
rc = 0;
}
}
if (pmp != 0) {
if (ahci_port_detect_pmp(ap)) {
pmp = 0;
retries = 0;
ahci_port_comreset(ap);
ahci_port_portreset_wait(ap);
goto retry;
}
}
err:
ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd);
ap->ap_saved_cmd = 0;
splx(s);
return (rc);
}
int
ahci_port_portreset(struct ahci_port *ap, int pmp)
{
ahci_port_portreset_start(ap);
ahci_port_portreset_wait(ap);
return (ahci_port_portreset_finish(ap, pmp));
}
int
ahci_port_detect_pmp(struct ahci_port *ap)
{
int count, pmp_rc, rc;
u_int32_t r, cmd;
struct ahci_cmd_hdr *cmd_slot;
struct ahci_ccb *ccb = NULL;
u_int8_t *fis = NULL;
if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) ||
!ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) {
return 0;
}
rc = 0;
pmp_rc = 0;
count = 2;
do {
DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
count);
if (ccb != NULL) {
ahci_put_pmp_ccb(ccb);
ccb = NULL;
}
ahci_port_stop(ap, 0);
ap->ap_state = AP_S_PMP_PROBE;
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
cmd |= AHCI_PREG_CMD_PMA;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
}
r = ahci_pread(ap, AHCI_PREG_SERR);
if (r & AHCI_PREG_SERR_DIAG_X)
ahci_pwrite(ap, AHCI_PREG_SERR,
AHCI_PREG_SERR_DIAG_X);
ahci_port_clo(ap);
r = ahci_pread(ap, AHCI_PREG_SERR);
ahci_pwrite(ap, AHCI_PREG_SERR, r);
if (ahci_port_start(ap, 0)) {
rc = EBUSY;
printf("%s: failed to start port, cannot probe PMP\n",
PORTNAME(ap));
break;
}
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
u_int32_t cap;
cap = ahci_read(ap->ap_sc, AHCI_REG_CAP);
printf("%s: CLO %s, need port reset\n",
PORTNAME(ap),
ISSET(cap, AHCI_REG_CAP_SCLO)
? "failed" : "unsupported");
pmp_rc = EBUSY;
break;
}
ccb = ahci_get_pmp_ccb(ap);
cmd_slot = ccb->ccb_cmd_hdr;
memset(ccb->ccb_cmd_table, 0,
sizeof(struct ahci_cmd_table));
fis = ccb->ccb_cmd_table->cfis;
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = SATA_PMP_CONTROL_PORT;
fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
htolem16(&cmd_slot->flags, 5 |
AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
AHCI_CMD_LIST_FLAG_PMP);
DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
PORTNAME(ap));
ccb->ccb_xa.state = ATA_S_PENDING;
if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
PORTNAME(ap));
pmp_rc = EBUSY;
continue;
}
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
printf("%s: port busy after first PMP probe FIS\n",
PORTNAME(ap));
}
ahci_flush_tfd(ap);
r = ahci_pread(ap, AHCI_PREG_SERR);
ahci_pwrite(ap, AHCI_PREG_SERR, r);
memset(ccb->ccb_cmd_table, 0,
sizeof(struct ahci_cmd_table));
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = SATA_PMP_CONTROL_PORT;
fis[15] = ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
htolem16(&cmd_slot->flags, 5 |
AHCI_CMD_LIST_FLAG_PMP);
DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
PORTNAME(ap));
ccb->ccb_xa.state = ATA_S_PENDING;
if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
"cmd failed\n", PORTNAME(ap));
pmp_rc = EBUSY;
continue;
}
fis[15] = 0;
pmp_rc = 0;
} while (--count);
if (ccb != NULL) {
ahci_put_pmp_ccb(ccb);
ccb = NULL;
}
if (ap->ap_state == AP_S_PMP_PROBE) {
ap->ap_state = AP_S_NORMAL;
}
if (pmp_rc == 0) {
if (ahci_port_signature(ap) != ATA_PORT_T_PM) {
DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
PORTNAME(ap));
pmp_rc = EBUSY;
} else {
DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
PORTNAME(ap));
}
}
if (pmp_rc == 0) {
if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
pmp_rc = EBUSY;
} else {
rc = 0;
}
}
if (pmp_rc != 0) {
DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
"the port\n", PORTNAME(ap));
ahci_port_stop(ap, 0);
ahci_port_clo(ap);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
cmd &= ~AHCI_PREG_CMD_PMA;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
ahci_pwrite(ap, AHCI_PREG_IE, 0);
ahci_port_stop(ap, 0);
if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
ahci_pwrite(ap, AHCI_PREG_SNTF, -1);
ahci_flush_tfd(ap);
ahci_pwrite(ap, AHCI_PREG_SERR, -1);
ahci_pwrite(ap, AHCI_PREG_IS, -1);
ahci_enable_interrupts(ap);
rc = pmp_rc;
}
return (rc);
}
void
ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
u_int32_t flags)
{
flags |= len - 1;
htolem64(&prd->dba, addr);
htolem32(&prd->flags, flags);
}
int
ahci_load_prdt(struct ahci_ccb *ccb)
{
struct ahci_port *ap = ccb->ccb_port;
struct ahci_softc *sc = ap->ap_sc;
struct ata_xfer *xa = &ccb->ccb_xa;
struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
bus_dmamap_t dmap = ccb->ccb_dmamap;
struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
int i, error;
if (xa->datalen == 0) {
ccb->ccb_cmd_hdr->prdtl = 0;
return (0);
}
error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
(xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
if (error != 0) {
printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
return (1);
}
for (i = 0; i < dmap->dm_nsegs - 1; i++) {
ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
dmap->dm_segs[i].ds_len, 0);
}
ahci_load_prdt_seg(&prdt[i],
dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0);
htolem16(&cmd_slot->prdtl, dmap->dm_nsegs);
bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
(xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
BUS_DMASYNC_PREWRITE);
return (0);
}
void
ahci_unload_prdt(struct ahci_ccb *ccb)
{
struct ahci_port *ap = ccb->ccb_port;
struct ahci_softc *sc = ap->ap_sc;
struct ata_xfer *xa = &ccb->ccb_xa;
bus_dmamap_t dmap = ccb->ccb_dmamap;
if (xa->datalen != 0) {
bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
(xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, dmap);
if (ccb->ccb_xa.flags & ATA_F_NCQ)
xa->resid = 0;
else
xa->resid = xa->datalen -
lemtoh32(&ccb->ccb_cmd_hdr->prdbc);
}
}
int
ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
{
struct ahci_port *ap = ccb->ccb_port;
int s;
s = splbio();
ahci_start(ccb);
do {
if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
1 << ccb->ccb_slot)) {
splx(s);
return (0);
}
if (ccb->ccb_xa.state == ATA_S_ERROR) {
DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
PORTNAME(ap), ccb->ccb_slot);
if (timeout_fn != NULL) {
timeout_fn(ccb);
}
splx(s);
return (1);
}
delay(1000);
} while (--timeout > 0);
if (timeout_fn != NULL)
timeout_fn(ccb);
splx(s);
return (1);
}
void
ahci_start(struct ahci_ccb *ccb)
{
struct ahci_port *ap = ccb->ccb_port;
struct ahci_softc *sc = ap->ap_sc;
ccb->ccb_cmd_hdr->prdbc = 0;
bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
ccb->ccb_slot * sizeof(struct ahci_cmd_table),
sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
if (ccb->ccb_xa.flags & ATA_F_NCQ) {
if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) ||
(ap->ap_sactive != 0 &&
ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
} else {
KASSERT(ap->ap_active_cnt == 0);
ap->ap_sactive |= (1 << ccb->ccb_slot);
ccb->ccb_xa.state = ATA_S_ONCHIP;
ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
}
} else {
if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
else if (ap->ap_active_cnt < 2) {
ap->ap_active |= 1 << ccb->ccb_slot;
ccb->ccb_xa.state = ATA_S_ONCHIP;
ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
ap->ap_active_cnt++;
}
}
}
void
ahci_issue_pending_ncq_commands(struct ahci_port *ap)
{
struct ahci_ccb *nextccb;
u_int32_t sact_change = 0;
KASSERT(ap->ap_active_cnt == 0);
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
return;
if (ap->ap_sactive != 0 &&
ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
return;
}
ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
do {
TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
sact_change |= 1 << nextccb->ccb_slot;
nextccb->ccb_xa.state = ATA_S_ONCHIP;
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
} while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) &&
(nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
ap->ap_sactive |= sact_change;
ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
}
void
ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
{
struct ahci_ccb *nextccb;
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
if (last_was_ncq) {
KASSERT(nextccb->ccb_xa.pmp_port !=
ap->ap_pmp_ncq_port);
} else {
ap->ap_active_cnt--;
}
if (ap->ap_active == 0)
ahci_issue_pending_ncq_commands(ap);
else
KASSERT(ap->ap_active_cnt == 1);
} else if (nextccb) {
if (ap->ap_sactive != 0 || last_was_ncq)
KASSERT(ap->ap_active_cnt == 0);
if (ap->ap_sactive != 0)
return;
do {
TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
ap->ap_active |= 1 << nextccb->ccb_slot;
nextccb->ccb_xa.state = ATA_S_ONCHIP;
ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
if (last_was_ncq)
ap->ap_active_cnt++;
if (ap->ap_active_cnt == 2)
break;
KASSERT(ap->ap_active_cnt == 1);
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
} while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
} else if (!last_was_ncq) {
KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
ap->ap_active_cnt--;
} else {
KASSERT(ap->ap_active_cnt == 0);
}
}
int
ahci_intr(void *arg)
{
struct ahci_softc *sc = arg;
u_int32_t is, ack = 0;
int port;
is = ahci_read(sc, AHCI_REG_IS);
if (is == 0 || is == 0xffffffff)
return (0);
ack = is;
#ifdef AHCI_COALESCE
if (is & sc->sc_ccc_mask) {
DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
DEVNAME(sc));
is &= ~sc->sc_ccc_mask;
is |= sc->sc_ccc_ports_cur;
}
#endif
while (is) {
port = ffs(is) - 1;
if (sc->sc_ports[port])
ahci_port_intr(sc->sc_ports[port],
AHCI_PREG_CI_ALL_SLOTS);
is &= ~(1 << port);
}
ahci_write(sc, AHCI_REG_IS, ack);
return (1);
}
u_int32_t
ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
{
struct ahci_softc *sc = ap->ap_sc;
u_int32_t is, ci_saved, ci_masked, processed = 0;
int slot, need_restart = 0;
int process_error = 0;
struct ahci_ccb *ccb;
volatile u_int32_t *active;
#ifdef DIAGNOSTIC
u_int32_t tmp;
#endif
is = ahci_pread(ap, AHCI_PREG_IS);
if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
ahci_pwrite(ap, AHCI_PREG_IS, is);
if (is)
DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
is, AHCI_PFMT_IS);
if (ap->ap_sactive) {
KASSERT(ap->ap_active == 0);
KASSERT(ap->ap_active_cnt == 0);
ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
active = &ap->ap_sactive;
} else {
ci_saved = ahci_pread(ap, AHCI_PREG_CI);
active = &ap->ap_active;
}
if (is & AHCI_PREG_IS_TFES) {
process_error = 1;
} else if (is & AHCI_PREG_IS_DHRS) {
u_int32_t tfd;
u_int32_t cmd;
u_int32_t serr;
tfd = ahci_pread(ap, AHCI_PREG_TFD);
cmd = ahci_pread(ap, AHCI_PREG_CMD);
serr = ahci_pread(ap, AHCI_PREG_SERR);
if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
(cmd & AHCI_PREG_CMD_CR) == 0) {
DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
" %b, DIAG: %b\n", PORTNAME(ap), tfd,
AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
AHCI_PFMT_SERR_DIAG);
process_error = 1;
} else {
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
}
}
if (process_error) {
u_int32_t tfd, serr;
int err_slot;
tfd = ahci_pread(ap, AHCI_PREG_TFD);
serr = ahci_pread(ap, AHCI_PREG_SERR);
if (ap->ap_sactive == 0) {
err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
AHCI_PREG_CMD));
if ((ci_saved & (1 << err_slot)) == 0) {
if (ap->ap_active_cnt == 1)
err_slot = ffs(ap->ap_active) - 1;
else
goto failall;
}
ccb = &ap->ap_ccbs[err_slot];
memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
sizeof(struct ata_fis_d2h));
} else
err_slot = -1;
DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
" %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
AHCI_PFMT_SERR_DIAG);
ahci_port_stop(ap, 0);
need_restart = 1;
ahci_pwrite(ap, AHCI_PREG_SERR, serr);
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
AHCI_PREG_IS_IFS);
is = ahci_pread(ap, AHCI_PREG_IS);
if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
if ((ap->ap_state == AP_S_PMP_PORT_PROBE) ||
(ap->ap_state == AP_S_ERROR_RECOVERY)) {
} else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
printf("%s: error on PMP port %d, idling "
"device\n", PORTNAME(ap),
ccb->ccb_xa.pmp_port);
if (ahci_pmp_port_softreset(ap,
ccb->ccb_xa.pmp_port) == 0) {
printf("%s: unable to softreset port "
"%d\n", PORTNAME(ap),
ccb->ccb_xa.pmp_port);
if (ahci_pmp_port_portreset(ap,
ccb->ccb_xa.pmp_port)) {
printf("%s: failed to port "
" reset %d, giving up on "
"it\n", PORTNAME(ap),
ccb->ccb_xa.pmp_port);
goto fatal;
}
}
} else {
printf("%s: attempting to idle device\n",
PORTNAME(ap));
if (ahci_port_softreset(ap)) {
printf("%s: failed to soft reset "
"device\n", PORTNAME(ap));
if (ahci_port_portreset(ap, 0)) {
printf("%s: failed to port "
"reset device, give up on "
"it\n", PORTNAME(ap));
goto fatal;
}
}
}
} else if (ap->ap_sactive) {
ahci_port_read_ncq_error(ap, &err_slot,
ap->ap_pmp_ncq_port);
if (err_slot < 0)
goto failall;
DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
PORTNAME(ap), err_slot);
ccb = &ap->ap_ccbs[err_slot];
if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
printf("%s: NCQ errored slot %d is idle"
" (%08x active)\n", PORTNAME(ap), err_slot,
ci_saved);
goto failall;
}
} else {
}
if (err_slot == -1) {
if (ahci_port_softreset(ap) != 0 &&
ahci_port_portreset(ap, 0) != 0) {
printf("%s: couldn't reset after NCQ error, "
"disabling device.\n", PORTNAME(ap));
goto fatal;
}
printf("%s: couldn't recover NCQ error, failing "
"all outstanding commands.\n", PORTNAME(ap));
goto failall;
}
ci_saved &= ~(1 << err_slot);
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
ccb->ccb_xa.state = ATA_S_ERROR;
#ifdef DIAGNOSTIC
if (ap->ap_sactive == 0) {
tmp = ci_saved;
if (tmp) {
slot = ffs(tmp) - 1;
tmp &= ~(1 << slot);
KASSERT(tmp == 0);
}
}
#endif
}
if ((ap->ap_state == AP_S_PMP_PROBE) &&
(ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) &&
(is & AHCI_PREG_IS_IPMS)) {
slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD));
DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
PORTNAME(ap), slot);
ccb = &ap->ap_ccbs[slot];
ccb->ccb_xa.state = ATA_S_ERROR;
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS);
is &= ~AHCI_PREG_IS_IPMS;
}
if ((is & AHCI_PREG_IS_IFS) ) {
DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
"port\n", PORTNAME(ap));
need_restart = 1;
ahci_pwrite(ap, AHCI_PREG_SERR, -1);
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
is &= ~AHCI_PREG_IS_IFS;
goto failall;
}
if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
PORTNAME(ap), is, AHCI_PFMT_IS);
goto fatal;
}
if (ap->ap_state == AP_S_FATAL_ERROR) {
fatal:
ap->ap_state = AP_S_FATAL_ERROR;
failall:
ahci_port_stop(ap, 1);
ci_masked = ci_saved & *active;
while (ci_masked) {
slot = ffs(ci_masked) - 1;
ccb = &ap->ap_ccbs[slot];
ci_masked &= ~(1 << slot);
ccb->ccb_xa.state = ATA_S_ERROR;
}
ci_saved &= ~*active;
if (ap->ap_state == AP_S_FATAL_ERROR)
need_restart = 0;
}
ci_masked = ~ci_saved & *active & ci_mask;
while (ci_masked) {
slot = ffs(ci_masked) - 1;
ccb = &ap->ap_ccbs[slot];
ci_masked &= ~(1 << slot);
DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
" (error)" : "");
bus_dmamap_sync(sc->sc_dmat,
AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
bus_dmamap_sync(sc->sc_dmat,
AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
ccb->ccb_slot * sizeof(struct ahci_cmd_table),
sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
bus_dmamap_sync(sc->sc_dmat,
AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
*active &= ~(1 << ccb->ccb_slot);
if (ccb->ccb_xa.state == ATA_S_ONCHIP &&
ccb->ccb_xa.flags & ATA_F_GET_RFIS) {
memcpy(&ccb->ccb_xa.rfis,
ap->ap_rfis->rfis,
sizeof(struct ata_fis_d2h));
}
processed |= 1 << ccb->ccb_slot;
ccb->ccb_done(ccb);
}
if (need_restart) {
ahci_port_start(ap, 0);
if (ci_saved) {
#ifdef DIAGNOSTIC
tmp = ci_saved;
while (tmp) {
slot = ffs(tmp) - 1;
tmp &= ~(1 << slot);
ccb = &ap->ap_ccbs[slot];
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
(!!ap->ap_sactive));
}
#endif
DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
"re-enabling%s slots %08x\n", PORTNAME(ap),
ap->ap_sactive ? " NCQ" : "", ci_saved);
if (ap->ap_sactive)
ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
}
}
return (processed);
}
struct ahci_ccb *
ahci_get_ccb(struct ahci_port *ap)
{
struct ahci_ccb *ccb;
mtx_enter(&ap->ap_ccb_mtx);
ccb = TAILQ_FIRST(&ap->ap_ccb_free);
if (ccb != NULL) {
KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
ccb->ccb_xa.state = ATA_S_SETUP;
}
mtx_leave(&ap->ap_ccb_mtx);
return (ccb);
}
void
ahci_put_ccb(struct ahci_ccb *ccb)
{
struct ahci_port *ap = ccb->ccb_port;
#ifdef DIAGNOSTIC
if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
ccb->ccb_xa.state != ATA_S_TIMEOUT &&
ccb->ccb_xa.state != ATA_S_ERROR) {
printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
"slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
ccb->ccb_slot);
}
#endif
ccb->ccb_xa.state = ATA_S_PUT;
mtx_enter(&ap->ap_ccb_mtx);
TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
mtx_leave(&ap->ap_ccb_mtx);
}
struct ahci_ccb *
ahci_get_err_ccb(struct ahci_port *ap)
{
struct ahci_ccb *err_ccb;
u_int32_t sact;
splassert(IPL_BIO);
sact = ahci_pread(ap, AHCI_PREG_SACT);
if (sact != 0)
printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
#ifdef DIAGNOSTIC
KASSERT(ap->ap_err_busy == 0);
ap->ap_err_busy = 1;
#endif
ap->ap_err_saved_active = ap->ap_active;
ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
ap->ap_err_saved_sactive = ap->ap_sactive;
ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
err_ccb = ap->ap_ccb_err;
err_ccb->ccb_xa.flags = 0;
err_ccb->ccb_xa.state = ATA_S_SETUP;
err_ccb->ccb_done = ahci_empty_done;
return (err_ccb);
}
void
ahci_put_err_ccb(struct ahci_ccb *ccb)
{
struct ahci_port *ap = ccb->ccb_port;
u_int32_t sact;
splassert(IPL_BIO);
#ifdef DIAGNOSTIC
KASSERT(ap->ap_err_busy);
#endif
sact = ahci_pread(ap, AHCI_PREG_SACT);
if (sact != 0)
printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact);
KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
KASSERT(ccb == ap->ap_ccb_err);
ap->ap_sactive = ap->ap_err_saved_sactive;
ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
ap->ap_active = ap->ap_err_saved_active;
#ifdef DIAGNOSTIC
ap->ap_err_busy = 0;
#endif
}
struct ahci_ccb *
ahci_get_pmp_ccb(struct ahci_port *ap)
{
struct ahci_ccb *ccb;
u_int32_t sact;
splassert(IPL_BIO);
sact = ahci_pread(ap, AHCI_PREG_SACT);
if (sact != 0)
printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
ccb = &ap->ap_ccbs[1];
KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
ccb->ccb_xa.flags = 0;
ccb->ccb_done = ahci_pmp_cmd_done;
mtx_enter(&ap->ap_ccb_mtx);
TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
mtx_leave(&ap->ap_ccb_mtx);
return ccb;
}
void
ahci_put_pmp_ccb(struct ahci_ccb *ccb)
{
struct ahci_port *ap = ccb->ccb_port;
u_int32_t sact;
KASSERT(ccb == &ap->ap_ccbs[1]);
sact = ahci_pread(ap, AHCI_PREG_SACT);
if (sact != 0)
printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact);
KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
ccb->ccb_xa.state = ATA_S_PUT;
mtx_enter(&ap->ap_ccb_mtx);
TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
mtx_leave(&ap->ap_ccb_mtx);
}
int
ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port)
{
struct ahci_ccb *ccb;
struct ahci_cmd_hdr *cmd_slot;
u_int32_t cmd;
struct ata_fis_h2d *fis;
int rc = EIO, oldstate;
DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
oldstate = ap->ap_state;
ap->ap_state = AP_S_ERROR_RECOVERY;
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
ahci_port_start(ap, 0);
ccb = ahci_get_err_ccb(ap);
ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
ccb->ccb_xa.data = ap->ap_err_scratch;
ccb->ccb_xa.datalen = 512;
cmd_slot = ccb->ccb_cmd_hdr;
memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
fis->type = ATA_FIS_TYPE_H2D;
fis->flags = ATA_H2D_FLAGS_CMD | pmp_port;
fis->command = ATA_C_READ_LOG_EXT;
fis->lba_low = 0x10;
fis->sector_count = 1;
fis->sector_count_exp = 0;
fis->lba_mid = 0;
fis->lba_mid_exp = 0;
fis->device = 0;
htolem16(&cmd_slot->flags, 5 |
(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
if (ahci_load_prdt(ccb) != 0) {
rc = ENOMEM;
goto err;
}
ccb->ccb_xa.state = ATA_S_PENDING;
if (ahci_poll(ccb, 1000, NULL) != 0 ||
ccb->ccb_xa.state == ATA_S_ERROR)
goto err;
rc = 0;
err:
if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
printf("%s: log page read failed, slot %d was still active.\n",
PORTNAME(ap), ccb->ccb_slot);
ahci_port_stop(ap, 0);
}
ahci_unload_prdt(ccb);
ahci_put_err_ccb(ccb);
if (rc == 0) {
struct ata_log_page_10h *log;
int err_slot;
log = (struct ata_log_page_10h *)ap->ap_err_scratch;
if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
printf("%s: read NCQ error page, but not an NCQ "
"error?\n", PORTNAME(ap));
rc = ESRCH;
} else {
*err_slotp = err_slot = log->err_regs.type &
ATA_LOG_10H_TYPE_TAG_MASK;
ccb = &ap->ap_ccbs[err_slot];
memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
sizeof(struct ata_fis_d2h));
ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
ccb->ccb_xa.rfis.flags = 0;
}
}
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
ap->ap_state = oldstate;
return (rc);
}
struct ahci_dmamem *
ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
{
struct ahci_dmamem *adm;
int nsegs;
adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO);
if (adm == NULL)
return (NULL);
adm->adm_size = size;
if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
goto admfree;
if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
goto destroy;
if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
&adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0)
goto free;
if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
NULL, BUS_DMA_NOWAIT) != 0)
goto unmap;
return (adm);
unmap:
bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
free:
bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
destroy:
bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
admfree:
free(adm, M_DEVBUF, sizeof(*adm));
return (NULL);
}
void
ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
{
bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
free(adm, M_DEVBUF, sizeof(*adm));
}
u_int32_t
ahci_read(struct ahci_softc *sc, bus_size_t r)
{
bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
BUS_SPACE_BARRIER_READ);
return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
}
void
ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
{
bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
BUS_SPACE_BARRIER_WRITE);
}
int
ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
u_int32_t target)
{
int i;
for (i = 0; i < 1000; i++) {
if ((ahci_read(sc, r) & mask) != target)
return (0);
delay(1000);
}
return (1);
}
u_int32_t
ahci_pread(struct ahci_port *ap, bus_size_t r)
{
bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
BUS_SPACE_BARRIER_READ);
return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
}
void
ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
{
bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
BUS_SPACE_BARRIER_WRITE);
}
int
ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
u_int32_t target, int n)
{
int i;
for (i = 0; i < n * 1000; i++) {
if ((ahci_pread(ap, r) & mask) == target)
return (0);
delay(1000);
}
return (1);
}
int
ahci_ata_probe(void *xsc, int port, int lun)
{
struct ahci_softc *sc = xsc;
struct ahci_port *ap = sc->sc_ports[port];
if (ap == NULL)
return (ATA_PORT_T_NONE);
if (lun != 0) {
int pmp_port = lun - 1;
if (pmp_port >= ap->ap_pmp_ports) {
return (ATA_PORT_T_NONE);
}
return (ahci_pmp_port_probe(ap, pmp_port));
} else {
return (ahci_port_signature(ap));
}
}
void
ahci_ata_free(void *xsc, int port, int lun)
{
}
struct ata_xfer *
ahci_ata_get_xfer(void *aaa_cookie, int port)
{
struct ahci_softc *sc = aaa_cookie;
struct ahci_port *ap = sc->sc_ports[port];
struct ahci_ccb *ccb;
ccb = ahci_get_ccb(ap);
if (ccb == NULL) {
DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
PORTNAME(ap));
return (NULL);
}
DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
PORTNAME(ap), ccb->ccb_slot);
return ((struct ata_xfer *)ccb);
}
void
ahci_ata_put_xfer(struct ata_xfer *xa)
{
struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
ahci_put_ccb(ccb);
}
void
ahci_ata_cmd(struct ata_xfer *xa)
{
struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
struct ahci_cmd_hdr *cmd_slot;
int s;
u_int16_t flags;
if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
goto failcmd;
ccb->ccb_done = ahci_ata_cmd_done;
cmd_slot = ccb->ccb_cmd_hdr;
flags = 5 ;
flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
if (xa->flags & ATA_F_WRITE)
flags |= AHCI_CMD_LIST_FLAG_W;
if (xa->flags & ATA_F_PACKET)
flags |= AHCI_CMD_LIST_FLAG_A;
htolem16(&cmd_slot->flags, flags);
if (ahci_load_prdt(ccb) != 0)
goto failcmd;
timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
xa->state = ATA_S_PENDING;
if (xa->flags & ATA_F_POLL)
ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
else {
s = splbio();
timeout_add_msec(&xa->stimeout, xa->timeout);
ahci_start(ccb);
splx(s);
}
return;
failcmd:
s = splbio();
xa->state = ATA_S_ERROR;
ata_complete(xa);
splx(s);
}
void
ahci_pmp_cmd_done(struct ahci_ccb *ccb)
{
struct ata_xfer *xa = &ccb->ccb_xa;
if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
ahci_issue_pending_commands(ccb->ccb_port,
xa->flags & ATA_F_NCQ);
xa->state = ATA_S_COMPLETE;
}
void
ahci_ata_cmd_done(struct ahci_ccb *ccb)
{
struct ata_xfer *xa = &ccb->ccb_xa;
timeout_del(&xa->stimeout);
if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
ahci_issue_pending_commands(ccb->ccb_port,
xa->flags & ATA_F_NCQ);
ahci_unload_prdt(ccb);
if (xa->state == ATA_S_ONCHIP)
xa->state = ATA_S_COMPLETE;
#ifdef DIAGNOSTIC
else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
"slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
ccb->ccb_slot);
#endif
if (xa->state != ATA_S_TIMEOUT)
ata_complete(xa);
}
void
ahci_ata_cmd_timeout(void *arg)
{
struct ahci_ccb *ccb = arg;
struct ata_xfer *xa = &ccb->ccb_xa;
struct ahci_port *ap = ccb->ccb_port;
int s, ccb_was_started, ncq_cmd;
volatile u_int32_t *active;
s = splbio();
ncq_cmd = (xa->flags & ATA_F_NCQ);
active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
if (ccb->ccb_xa.state == ATA_S_PENDING) {
DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
"before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
ccb_was_started = 0;
} else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
1 << ccb->ccb_slot)) {
DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
"command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
goto ret;
} else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
"handled%s\n", PORTNAME(ap), ccb->ccb_slot,
ISSET(*active, 1 << ccb->ccb_slot) ?
" but slot is still active?" : ".");
goto ret;
} else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
1 << ccb->ccb_slot)) {
DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
"IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
ccb->ccb_slot);
*active &= ~(1 << ccb->ccb_slot);
ccb->ccb_done(ccb);
goto ret;
} else {
ccb_was_started = 1;
}
ccb->ccb_xa.state = ATA_S_TIMEOUT;
*active &= ~(1 << ccb->ccb_slot);
DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
ccb->ccb_done(ccb);
if (ccb_was_started) {
DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
"in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
!= 0) {
printf("%s: failed to reset port during timeout "
"handling, disabling it\n", PORTNAME(ap));
ap->ap_state = AP_S_FATAL_ERROR;
}
if (*active) {
DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
"%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
*active);
if (ncq_cmd)
ahci_pwrite(ap, AHCI_PREG_SACT, *active);
ahci_pwrite(ap, AHCI_PREG_CI, *active);
}
}
DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
if (ccb_was_started)
ahci_issue_pending_commands(ap, ncq_cmd);
else if (ap->ap_active == 0)
ahci_issue_pending_ncq_commands(ap);
DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
ata_complete(xa);
DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
ret:
splx(s);
}
void
ahci_empty_done(struct ahci_ccb *ccb)
{
if (ccb->ccb_xa.state != ATA_S_ERROR)
ccb->ccb_xa.state = ATA_S_COMPLETE;
}
int
ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap)
{
struct ahci_ccb *ccb;
struct ata_fis_h2d *fis;
int error;
ccb = ahci_get_pmp_ccb(ap);
ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS;
ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
ccb->ccb_xa.state = ATA_S_PENDING;
memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
fis->type = ATA_FIS_TYPE_H2D;
fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
fis->command = ATA_C_READ_PM;
fis->features = which;
fis->device = target | ATA_H2D_DEVICE_LBA;
fis->control = ATA_FIS_CONTROL_4BIT;
if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
error = 1;
} else {
*datap = ccb->ccb_xa.rfis.sector_count |
(ccb->ccb_xa.rfis.lba_low << 8) |
(ccb->ccb_xa.rfis.lba_mid << 16) |
(ccb->ccb_xa.rfis.lba_high << 24);
error = 0;
}
ahci_put_pmp_ccb(ccb);
return (error);
}
int
ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
{
struct ahci_ccb *ccb;
struct ata_fis_h2d *fis;
int error;
ccb = ahci_get_pmp_ccb(ap);
ccb->ccb_xa.flags = ATA_F_POLL;
ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
ccb->ccb_xa.state = ATA_S_PENDING;
memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
fis->type = ATA_FIS_TYPE_H2D;
fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
fis->command = ATA_C_WRITE_PM;
fis->features = which;
fis->device = target | ATA_H2D_DEVICE_LBA;
fis->sector_count = (u_int8_t)data;
fis->lba_low = (u_int8_t)(data >> 8);
fis->lba_mid = (u_int8_t)(data >> 16);
fis->lba_high = (u_int8_t)(data >> 24);
fis->control = ATA_FIS_CONTROL_4BIT;
error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
ahci_put_pmp_ccb(ccb);
return (error);
}
int
ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap)
{
int error;
error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap);
if (error == 0)
error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1);
if (error)
*datap = 0;
return (error);
}
int
ahci_pmp_identify(struct ahci_port *ap, int *ret_nports)
{
u_int32_t chipid;
u_int32_t rev;
u_int32_t nports;
u_int32_t features;
u_int32_t enabled;
int s;
s = splbio();
if (ahci_pmp_read(ap, 15, 0, &chipid) ||
ahci_pmp_read(ap, 15, 1, &rev) ||
ahci_pmp_read(ap, 15, 2, &nports) ||
ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) ||
ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) {
printf("%s: port multiplier identification failed\n",
PORTNAME(ap));
splx(s);
return (1);
}
splx(s);
nports &= 0x0F;
if (chipid == 0x37261095) {
nports--;
}
printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
"features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev,
SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled,
SATA_PFMT_PM_FEA);
*ret_nports = nports;
return (0);
}
#ifdef HIBERNATE
void
ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb)
{
ccb->ccb_cmd_hdr->prdbc = 0;
ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
}
int
ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb)
{
u_int32_t is, ci_saved;
int process_error = 0;
is = ahci_pread(ap, AHCI_PREG_IS);
ci_saved = ahci_pread(ap, AHCI_PREG_CI);
if (is & AHCI_PREG_IS_DHRS) {
u_int32_t tfd;
u_int32_t cmd;
tfd = ahci_pread(ap, AHCI_PREG_TFD);
cmd = ahci_pread(ap, AHCI_PREG_CMD);
if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
(cmd & AHCI_PREG_CMD_CR) == 0) {
process_error = 1;
} else {
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
}
} else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
process_error = 1;
}
if (process_error) {
ahci_port_stop(ap, 0);
return (EIO);
}
if (ci_saved & (1 << ccb->ccb_slot)) {
return (EAGAIN);
}
return (0);
}
void
ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
{
struct ata_xfer *xa = &ccb->ccb_xa;
struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
int i;
paddr_t data_phys;
u_int64_t data_bus_phys;
vaddr_t data_addr;
size_t seglen;
size_t buflen;
if (xa->datalen == 0) {
ccb->ccb_cmd_hdr->prdtl = 0;
return;
}
buflen = xa->datalen;
data_addr = (vaddr_t)xa->data;
for (i = 0; buflen > 0; i++) {
pmap_extract(pmap_kernel(), data_addr, &data_phys);
data_bus_phys = data_phys;
seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET);
if (buflen < seglen)
seglen = buflen;
ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
data_addr += seglen;
buflen -= seglen;
}
htolem16(&cmd_slot->prdtl, i);
}
int
ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
int op, void *page)
{
struct {
struct ahci_cmd_hdr cmd_hdr[32];
struct ahci_rfis rfis;
struct ahci_cmd_table cmd_table;
struct ahci_port *ap;
struct ahci_ccb ccb_buf;
struct ahci_ccb *ccb;
struct ahci_cmd_hdr *hdr_buf;
int pmp_port;
daddr_t poffset;
size_t psize;
} *my = page;
struct ata_fis_h2d *fis;
u_int32_t sector_count;
struct ahci_cmd_hdr *cmd_slot;
int rc;
int timeout;
u_int16_t flags;
if (op == HIB_INIT) {
struct device *disk;
struct device *scsibus;
struct ahci_softc *sc;
extern struct cfdriver sd_cd;
struct scsi_link *link;
struct scsibus_softc *bus_sc;
int port;
paddr_t page_phys;
u_int64_t item_phys;
u_int32_t cmd;
my->poffset = blkno;
my->psize = size;
disk = disk_lookup(&sd_cd, DISKUNIT(dev));
scsibus = disk->dv_parent;
sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
port = -1;
bus_sc = (struct scsibus_softc *)scsibus;
SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) {
if (link->device_softc == disk) {
port = link->target;
if (link->lun > 0)
my->pmp_port = link->lun - 1;
else
my->pmp_port = 0;
break;
}
}
if (port == -1) {
return (EIO);
}
my->ap = sc->sc_ports[port];
if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) {
return (EIO);
}
if (ahci_port_stop(my->ap, 1)) {
return (EIO);
}
ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0);
pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys);
item_phys = page_phys + ((void *)&my->rfis - page);
ahci_pwrite(my->ap, AHCI_PREG_FBU,
(u_int32_t)(item_phys >> 32));
ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys);
cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD |
AHCI_PREG_CMD_SUD;
ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd |
AHCI_PREG_CMD_ICC_ACTIVE);
cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
return (EIO);
}
my->ccb = &my->ccb_buf;
my->ccb->ccb_slot = 0;
my->ccb->ccb_port = my->ap;
item_phys = page_phys + ((void *)&my->cmd_hdr - page);
ahci_pwrite(my->ap, AHCI_PREG_CLBU,
(u_int32_t)(item_phys >> 32));
ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys);
my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
pmap_extract(pmap_kernel(),
(vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table),
&page_phys);
item_phys = page_phys;
#if 0
my->ccb->ccb_cmd_table = &my->cmd_table;
item_phys = page_phys + ((void *)&my->cmd_table - page);
#endif
htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys);
my->ccb->ccb_xa.fis =
(struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
my->ccb->ccb_xa.tag = 0;
ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
if (ahci_port_start(my->ap, 0)) {
return (EIO);
}
ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap,
AHCI_PREG_IS));
ahci_write(sc, AHCI_REG_IS, 1 << port);
ahci_enable_interrupts(my->ap);
return (0);
} else if (op == HIB_DONE) {
ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME);
return (0);
}
if (blkno > my->psize)
return (E2BIG);
blkno += my->poffset;
sector_count = size / 512;
my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ;
fis = my->ccb->ccb_xa.fis;
fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port;
fis->lba_low = blkno & 0xff;
fis->lba_mid = (blkno >> 8) & 0xff;
fis->lba_high = (blkno >> 16) & 0xff;
if (sector_count > 0x100 || blkno > 0xfffffff) {
fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT :
ATA_C_READDMA_EXT;
fis->device = ATA_H2D_DEVICE_LBA;
fis->lba_low_exp = (blkno >> 24) & 0xff;
fis->lba_mid_exp = (blkno >> 32) & 0xff;
fis->lba_high_exp = (blkno >> 40) & 0xff;
fis->sector_count = sector_count & 0xff;
fis->sector_count_exp = (sector_count >> 8) & 0xff;
} else {
fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA;
fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f);
fis->sector_count = sector_count & 0xff;
}
my->ccb->ccb_xa.data = (void *)addr;
my->ccb->ccb_xa.datalen = size;
my->ccb->ccb_xa.pmp_port = my->pmp_port;
my->ccb->ccb_xa.flags |= ATA_F_POLL;
cmd_slot = my->ccb->ccb_cmd_hdr;
flags = 5;
flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
if (op == HIB_W)
flags |= AHCI_CMD_LIST_FLAG_W;
htolem16(&cmd_slot->flags, flags);
ahci_hibernate_load_prdt(my->ccb);
ahci_hibernate_io_start(my->ap, my->ccb);
timeout = 1000000;
while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) {
delay(1);
timeout--;
if (timeout == 0) {
return (EIO);
}
}
return (0);
}
#endif
