#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/mutex.h>
#include <sys/systm.h>
#include <x86/apicreg.h>
#include <dev/ic/i8259.h>
#include <machine/vmm.h>
#include "vmm_lapic.h"
#include "vioapic.h"
#include "vatpic.h"
#define VATPIC_LOCK(vatpic) mutex_enter(&((vatpic)->lock))
#define VATPIC_UNLOCK(vatpic) mutex_exit(&((vatpic)->lock))
#define VATPIC_LOCKED(vatpic) MUTEX_HELD(&((vatpic)->lock))
#define IRQ_BASE_MASK 0xf8
enum irqstate {
IRQSTATE_ASSERT,
IRQSTATE_DEASSERT,
IRQSTATE_PULSE
};
enum icw_state {
IS_ICW1 = 0,
IS_ICW2,
IS_ICW3,
IS_ICW4,
};
struct atpic {
enum icw_state icw_state;
bool ready;
bool auto_eoi;
bool poll;
bool rotate;
bool special_full_nested;
bool read_isr_next;
bool intr_raised;
bool special_mask_mode;
uint8_t reg_irr;
uint8_t reg_isr;
uint8_t reg_imr;
uint8_t irq_base;
uint8_t lowprio;
uint8_t elc;
uint_t acnt[8];
};
struct atpic_stats {
uint64_t as_interrupts;
uint64_t as_saturate_low;
uint64_t as_saturate_high;
};
struct vatpic {
struct vm *vm;
kmutex_t lock;
struct atpic atpic[2];
struct atpic_stats stats;
};
#define ATPIC_PIN_FOREACH(pinvar, atpic, tmpvar) \
for (tmpvar = 0, pinvar = (atpic->lowprio + 1) & 0x7; \
tmpvar < 8; \
tmpvar++, pinvar = (pinvar + 1) & 0x7)
static int vatpic_set_pinstate(struct vatpic *vatpic, int pin, bool newstate);
static __inline bool
master_atpic(struct vatpic *vatpic, struct atpic *atpic)
{
if (atpic == &vatpic->atpic[0])
return (true);
else
return (false);
}
static __inline int
vatpic_get_highest_isrpin(struct atpic *atpic)
{
int bit, pin;
int i;
ATPIC_PIN_FOREACH(pin, atpic, i) {
bit = (1 << pin);
if (atpic->reg_isr & bit) {
if (atpic->special_mask_mode &&
(atpic->reg_imr & bit) != 0) {
continue;
} else {
return (pin);
}
}
}
return (-1);
}
static __inline int
vatpic_get_highest_irrpin(struct atpic *atpic)
{
int serviced;
int bit, pin, tmp;
serviced = atpic->reg_isr;
if (atpic->special_full_nested)
serviced &= ~(1 << 2);
if (atpic->special_mask_mode)
serviced = 0;
ATPIC_PIN_FOREACH(pin, atpic, tmp) {
bit = 1 << pin;
if ((serviced & bit) != 0)
break;
if ((atpic->reg_irr & bit) != 0 && (atpic->reg_imr & bit) == 0)
return (pin);
}
return (-1);
}
static void
vatpic_notify_intr(struct vatpic *vatpic)
{
struct atpic *atpic;
int pin;
ASSERT(VATPIC_LOCKED(vatpic));
atpic = &vatpic->atpic[1];
if (!atpic->intr_raised &&
(pin = vatpic_get_highest_irrpin(atpic)) != -1) {
atpic->intr_raised = true;
if (vatpic_set_pinstate(vatpic, 2, true) == 0) {
(void) vatpic_set_pinstate(vatpic, 2, false);
}
} else {
}
atpic = &vatpic->atpic[0];
if (!atpic->intr_raised &&
(pin = vatpic_get_highest_irrpin(atpic)) != -1) {
atpic->intr_raised = true;
(void) lapic_set_local_intr(vatpic->vm, -1, APIC_LVT_LINT0);
(void) vioapic_pulse_irq(vatpic->vm, 0);
vatpic->stats.as_interrupts++;
} else {
}
}
static int
vatpic_icw1(struct vatpic *vatpic, struct atpic *atpic, uint8_t val)
{
atpic->ready = false;
atpic->icw_state = IS_ICW1;
atpic->reg_irr = 0;
atpic->reg_imr = 0;
atpic->lowprio = 7;
atpic->read_isr_next = false;
atpic->poll = false;
atpic->special_mask_mode = false;
if ((val & ICW1_SNGL) != 0) {
return (-1);
}
if ((val & ICW1_IC4) == 0) {
return (-1);
}
atpic->icw_state = IS_ICW2;
return (0);
}
static int
vatpic_icw2(struct vatpic *vatpic, struct atpic *atpic, uint8_t val)
{
atpic->irq_base = val & IRQ_BASE_MASK;
atpic->icw_state = IS_ICW3;
return (0);
}
static int
vatpic_icw3(struct vatpic *vatpic, struct atpic *atpic, uint8_t val)
{
atpic->icw_state = IS_ICW4;
return (0);
}
static int
vatpic_icw4(struct vatpic *vatpic, struct atpic *atpic, uint8_t val)
{
if ((val & ICW4_8086) == 0) {
return (-1);
}
atpic->auto_eoi = (val & ICW4_AEOI) != 0;
if (master_atpic(vatpic, atpic)) {
atpic->special_full_nested = (val & ICW4_SFNM) != 0;
}
atpic->icw_state = IS_ICW1;
atpic->ready = true;
return (0);
}
static int
vatpic_ocw1(struct vatpic *vatpic, struct atpic *atpic, uint8_t val)
{
atpic->reg_imr = val;
return (0);
}
static int
vatpic_ocw2(struct vatpic *vatpic, struct atpic *atpic, uint8_t val)
{
atpic->rotate = (val & OCW2_R) != 0;
if ((val & OCW2_EOI) != 0) {
int isr_bit;
if ((val & OCW2_SL) != 0) {
isr_bit = val & 0x7;
} else {
isr_bit = vatpic_get_highest_isrpin(atpic);
}
if (isr_bit != -1) {
atpic->reg_isr &= ~(1 << isr_bit);
if (atpic->rotate)
atpic->lowprio = isr_bit;
}
} else if ((val & OCW2_SL) != 0 && atpic->rotate) {
atpic->lowprio = val & 0x7;
}
return (0);
}
static int
vatpic_ocw3(struct vatpic *vatpic, struct atpic *atpic, uint8_t val)
{
if ((val & OCW3_ESMM) != 0) {
atpic->special_mask_mode = (val & OCW3_SMM) != 0;
}
if ((val & OCW3_RR) != 0) {
atpic->read_isr_next = (val & OCW3_RIS) != 0;
}
if ((val & OCW3_P) != 0) {
atpic->poll = true;
}
return (0);
}
static int
vatpic_set_pinstate(struct vatpic *vatpic, int pin, bool newstate)
{
struct atpic *atpic;
uint_t oldcnt, newcnt;
int err = 0;
VERIFY(pin >= 0 && pin < 16);
ASSERT(VATPIC_LOCKED(vatpic));
const int lpin = pin & 0x7;
atpic = &vatpic->atpic[pin >> 3];
oldcnt = newcnt = atpic->acnt[lpin];
if (newstate) {
if (newcnt != UINT_MAX) {
newcnt++;
} else {
err = E2BIG;
DTRACE_PROBE2(vatpic__sat_high, struct vatpic *, vatpic,
int, pin);
vatpic->stats.as_saturate_high++;
}
} else {
if (newcnt != 0) {
newcnt--;
} else {
err = ERANGE;
DTRACE_PROBE2(vatpic__sat_low, struct vatpic *, vatpic,
int, pin);
vatpic->stats.as_saturate_low++;
}
}
atpic->acnt[lpin] = newcnt;
const bool level = ((atpic->elc & (1 << (lpin))) != 0);
if ((oldcnt == 0 && newcnt == 1) || (newcnt > 0 && level == true)) {
DTRACE_PROBE2(vatpic__assert, struct vatpic *, vatpic,
int, pin);
atpic->reg_irr |= (1 << lpin);
} else if (oldcnt == 1 && newcnt == 0) {
DTRACE_PROBE2(vatpic__deassert, struct vatpic *, vatpic,
int, pin);
if (level) {
atpic->reg_irr &= ~(1 << lpin);
}
}
vatpic_notify_intr(vatpic);
return (err);
}
static int
vatpic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
{
struct vatpic *vatpic;
struct atpic *atpic;
int err = 0;
if (irq < 0 || irq > 15)
return (EINVAL);
vatpic = vm_atpic(vm);
atpic = &vatpic->atpic[irq >> 3];
if (!atpic->ready)
return (0);
VATPIC_LOCK(vatpic);
switch (irqstate) {
case IRQSTATE_ASSERT:
err = vatpic_set_pinstate(vatpic, irq, true);
break;
case IRQSTATE_DEASSERT:
err = vatpic_set_pinstate(vatpic, irq, false);
break;
case IRQSTATE_PULSE:
err = vatpic_set_pinstate(vatpic, irq, true);
if (err == 0) {
err = vatpic_set_pinstate(vatpic, irq, false);
}
break;
default:
panic("vatpic_set_irqstate: invalid irqstate %d", irqstate);
}
VATPIC_UNLOCK(vatpic);
return (err);
}
int
vatpic_assert_irq(struct vm *vm, int irq)
{
return (vatpic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
}
int
vatpic_deassert_irq(struct vm *vm, int irq)
{
return (vatpic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
}
int
vatpic_pulse_irq(struct vm *vm, int irq)
{
return (vatpic_set_irqstate(vm, irq, IRQSTATE_PULSE));
}
int
vatpic_set_irq_trigger(struct vm *vm, int irq, enum vm_intr_trigger trigger)
{
if (irq < 0 || irq > 15)
return (EINVAL);
if (trigger == LEVEL_TRIGGER) {
switch (irq) {
case 0:
case 1:
case 2:
case 8:
case 13:
return (EINVAL);
}
}
struct vatpic *vatpic = vm_atpic(vm);
struct atpic *atpic = &vatpic->atpic[irq >> 3];
const int pin = irq & 0x7;
VATPIC_LOCK(vatpic);
if (trigger == LEVEL_TRIGGER) {
atpic->elc |= (1 << pin);
} else {
atpic->elc &= ~(1 << pin);
}
VATPIC_UNLOCK(vatpic);
return (0);
}
void
vatpic_pending_intr(struct vm *vm, int *vecptr)
{
struct vatpic *vatpic;
struct atpic *atpic;
int pin;
vatpic = vm_atpic(vm);
atpic = &vatpic->atpic[0];
VATPIC_LOCK(vatpic);
pin = vatpic_get_highest_irrpin(atpic);
if (pin == 2) {
atpic = &vatpic->atpic[1];
pin = vatpic_get_highest_irrpin(atpic);
}
if (pin == -1)
pin = 7;
KASSERT(pin >= 0 && pin <= 7, ("%s: invalid pin %d", __func__, pin));
*vecptr = atpic->irq_base + pin;
VATPIC_UNLOCK(vatpic);
}
static void
vatpic_pin_accepted(struct atpic *atpic, int pin)
{
ASSERT(pin >= 0 && pin < 8);
atpic->intr_raised = false;
if (atpic->acnt[pin] == 0)
atpic->reg_irr &= ~(1 << pin);
if (atpic->auto_eoi) {
if (atpic->rotate)
atpic->lowprio = pin;
} else {
atpic->reg_isr |= (1 << pin);
}
}
void
vatpic_intr_accepted(struct vm *vm, int vector)
{
struct vatpic *vatpic;
int pin;
vatpic = vm_atpic(vm);
VATPIC_LOCK(vatpic);
pin = vector & 0x7;
if ((vector & IRQ_BASE_MASK) == vatpic->atpic[1].irq_base) {
vatpic_pin_accepted(&vatpic->atpic[1], pin);
vatpic_pin_accepted(&vatpic->atpic[0], 2);
} else {
vatpic_pin_accepted(&vatpic->atpic[0], pin);
}
vatpic_notify_intr(vatpic);
VATPIC_UNLOCK(vatpic);
}
static int
vatpic_read(struct vatpic *vatpic, struct atpic *atpic, bool in, int port,
int bytes, uint32_t *eax)
{
int pin;
VATPIC_LOCK(vatpic);
if (atpic->poll) {
atpic->poll = false;
pin = vatpic_get_highest_irrpin(atpic);
if (pin >= 0) {
vatpic_pin_accepted(atpic, pin);
*eax = 0x80 | pin;
} else {
*eax = 0;
}
} else {
if (port & ICU_IMR_OFFSET) {
*eax = atpic->reg_imr;
} else {
if (atpic->read_isr_next) {
*eax = atpic->reg_isr;
} else {
*eax = atpic->reg_irr;
}
}
}
VATPIC_UNLOCK(vatpic);
return (0);
}
static int
vatpic_write(struct vatpic *vatpic, struct atpic *atpic, bool in, int port,
int bytes, uint32_t *eax)
{
int error;
uint8_t val;
error = 0;
val = *eax;
VATPIC_LOCK(vatpic);
if (port & ICU_IMR_OFFSET) {
switch (atpic->icw_state) {
case IS_ICW2:
error = vatpic_icw2(vatpic, atpic, val);
break;
case IS_ICW3:
error = vatpic_icw3(vatpic, atpic, val);
break;
case IS_ICW4:
error = vatpic_icw4(vatpic, atpic, val);
break;
default:
error = vatpic_ocw1(vatpic, atpic, val);
break;
}
} else {
if (val & (1 << 4))
error = vatpic_icw1(vatpic, atpic, val);
if (atpic->ready) {
if (val & (1 << 3))
error = vatpic_ocw3(vatpic, atpic, val);
else
error = vatpic_ocw2(vatpic, atpic, val);
}
}
if (atpic->ready)
vatpic_notify_intr(vatpic);
VATPIC_UNLOCK(vatpic);
return (error);
}
int
vatpic_master_handler(void *arg, bool in, uint16_t port, uint8_t bytes,
uint32_t *eax)
{
struct vatpic *vatpic = arg;
struct atpic *atpic = &vatpic->atpic[0];
if (bytes != 1)
return (-1);
if (in) {
return (vatpic_read(vatpic, atpic, in, port, bytes, eax));
}
return (vatpic_write(vatpic, atpic, in, port, bytes, eax));
}
int
vatpic_slave_handler(void *arg, bool in, uint16_t port, uint8_t bytes,
uint32_t *eax)
{
struct vatpic *vatpic = arg;
struct atpic *atpic = &vatpic->atpic[1];
if (bytes != 1)
return (-1);
if (in) {
return (vatpic_read(vatpic, atpic, in, port, bytes, eax));
}
return (vatpic_write(vatpic, atpic, in, port, bytes, eax));
}
static const uint8_t vatpic_elc_mask[2] = {
0xf8,
0xde
};
int
vatpic_elc_handler(void *arg, bool in, uint16_t port, uint8_t bytes,
uint32_t *eax)
{
struct vatpic *vatpic = arg;
struct atpic *atpic = NULL;
uint8_t elc_mask = 0;
switch (port) {
case IO_ELCR1:
atpic = &vatpic->atpic[0];
elc_mask = vatpic_elc_mask[0];
break;
case IO_ELCR2:
atpic = &vatpic->atpic[1];
elc_mask = vatpic_elc_mask[1];
break;
default:
return (-1);
}
if (bytes != 1)
return (-1);
VATPIC_LOCK(vatpic);
if (in) {
*eax = atpic->elc;
} else {
atpic->elc = *eax & elc_mask;
}
VATPIC_UNLOCK(vatpic);
return (0);
}
struct vatpic *
vatpic_init(struct vm *vm)
{
struct vatpic *vatpic;
vatpic = kmem_zalloc(sizeof (struct vatpic), KM_SLEEP);
vatpic->vm = vm;
mutex_init(&vatpic->lock, NULL, MUTEX_ADAPTIVE, NULL);
return (vatpic);
}
void
vatpic_cleanup(struct vatpic *vatpic)
{
mutex_destroy(&vatpic->lock);
kmem_free(vatpic, sizeof (*vatpic));
}
static int
vatpic_data_read(void *datap, const vmm_data_req_t *req)
{
VERIFY3U(req->vdr_class, ==, VDC_ATPIC);
VERIFY3U(req->vdr_version, ==, 1);
VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_atpic_v1));
struct vatpic *vatpic = datap;
struct vdi_atpic_v1 *out = req->vdr_data;
VATPIC_LOCK(vatpic);
for (uint_t i = 0; i < 2; i++) {
const struct atpic *src = &vatpic->atpic[i];
struct vdi_atpic_chip_v1 *chip = &out->va_chip[i];
chip->vac_icw_state = src->icw_state;
chip->vac_status =
(src->ready ? (1 << 0) : 0) |
(src->auto_eoi ? (1 << 1) : 0) |
(src->poll ? (1 << 2) : 0) |
(src->rotate ? (1 << 3) : 0) |
(src->special_full_nested ? (1 << 4) : 0) |
(src->read_isr_next ? (1 << 5) : 0) |
(src->intr_raised ? (1 << 6) : 0) |
(src->special_mask_mode ? (1 << 7) : 0);
chip->vac_reg_irr = src->reg_irr;
chip->vac_reg_isr = src->reg_isr;
chip->vac_reg_imr = src->reg_imr;
chip->vac_irq_base = src->irq_base;
chip->vac_lowprio = src->lowprio;
chip->vac_elc = src->elc;
for (uint_t j = 0; j < 8; j++) {
chip->vac_level[j] = src->acnt[j];
}
}
VATPIC_UNLOCK(vatpic);
return (0);
}
static bool
vatpic_data_validate(const struct vdi_atpic_v1 *src)
{
for (uint_t i = 0; i < 2; i++) {
const struct vdi_atpic_chip_v1 *chip = &src->va_chip[i];
if (chip->vac_icw_state > IS_ICW4) {
return (false);
}
if ((chip->vac_elc & ~vatpic_elc_mask[i]) != 0) {
return (false);
}
}
return (true);
}
static int
vatpic_data_write(void *datap, const vmm_data_req_t *req)
{
VERIFY3U(req->vdr_class, ==, VDC_ATPIC);
VERIFY3U(req->vdr_version, ==, 1);
VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_atpic_v1));
struct vatpic *vatpic = datap;
const struct vdi_atpic_v1 *src = req->vdr_data;
if (!vatpic_data_validate(src)) {
return (EINVAL);
}
VATPIC_LOCK(vatpic);
for (uint_t i = 0; i < 2; i++) {
const struct vdi_atpic_chip_v1 *chip = &src->va_chip[i];
struct atpic *out = &vatpic->atpic[i];
out->icw_state = chip->vac_icw_state;
out->ready = (chip->vac_status & (1 << 0)) != 0;
out->auto_eoi = (chip->vac_status & (1 << 1)) != 0;
out->poll = (chip->vac_status & (1 << 2)) != 0;
out->rotate = (chip->vac_status & (1 << 3)) != 0;
out->special_full_nested = (chip->vac_status & (1 << 4)) != 0;
out->read_isr_next = (chip->vac_status & (1 << 5)) != 0;
out->intr_raised = (chip->vac_status & (1 << 6)) != 0;
out->special_mask_mode = (chip->vac_status & (1 << 7)) != 0;
out->reg_irr = chip->vac_reg_irr;
out->reg_isr = chip->vac_reg_isr;
out->reg_imr = chip->vac_reg_imr;
out->irq_base = chip->vac_irq_base;
out->lowprio = chip->vac_lowprio;
out->elc = chip->vac_elc;
for (uint_t j = 0; j < 8; j++) {
out->acnt[j] = chip->vac_level[j];
}
}
VATPIC_UNLOCK(vatpic);
return (0);
}
static const vmm_data_version_entry_t atpic_v1 = {
.vdve_class = VDC_ATPIC,
.vdve_version = 1,
.vdve_len_expect = sizeof (struct vdi_atpic_v1),
.vdve_readf = vatpic_data_read,
.vdve_writef = vatpic_data_write,
};
VMM_DATA_VERSION(atpic_v1);