/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2016 Anish Gupta (anish@freebsd.org)
 * Copyright (c) 2021 The FreeBSD Foundation
 *
 * Portions of this software were developed by Ka Ho Ng
 * under sponsorship from the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _AMDVI_PRIV_H_
#define _AMDVI_PRIV_H_

#include <contrib/dev/acpica/include/acpi.h>

#define BIT(n)                  (1ULL << (n))
/* Return value of bits[n:m] where n and (n >= ) m are bit positions. */
#define REG_BITS(x, n, m)       (((x) >> (m)) &                 \
                                ((1 << (((n) - (m)) + 1)) - 1))

/*
 * IOMMU PCI capability.
 */
#define AMDVI_PCI_CAP_IOTLB     BIT(0)  /* IOTLB is supported. */
#define AMDVI_PCI_CAP_HT        BIT(1)  /* HyperTransport tunnel support. */
#define AMDVI_PCI_CAP_NPCACHE   BIT(2)  /* Not present page cached. */
#define AMDVI_PCI_CAP_EFR       BIT(3)  /* Extended features. */
#define AMDVI_PCI_CAP_EXT       BIT(4)  /* Miscellaneous information reg. */

/*
 * IOMMU extended features.
 */
#define AMDVI_EX_FEA_PREFSUP    BIT(0)  /* Prefetch command support. */
#define AMDVI_EX_FEA_PPRSUP     BIT(1)  /* PPR support */
#define AMDVI_EX_FEA_XTSUP      BIT(2)  /* Reserved */
#define AMDVI_EX_FEA_NXSUP      BIT(3)  /* No-execute. */
#define AMDVI_EX_FEA_GTSUP      BIT(4)  /* Guest translation support. */
#define AMDVI_EX_FEA_EFRW       BIT(5)  /* Reserved */
#define AMDVI_EX_FEA_IASUP      BIT(6)  /* Invalidate all command supp. */
#define AMDVI_EX_FEA_GASUP      BIT(7)  /* Guest APIC or AVIC support. */
#define AMDVI_EX_FEA_HESUP      BIT(8)  /* Hardware Error. */
#define AMDVI_EX_FEA_PCSUP      BIT(9)  /* Performance counters support. */
/* XXX: add more EFER bits. */

/*
 * Device table entry or DTE
 * NOTE: Must be 256-bits/32 bytes aligned.
 */
struct amdvi_dte {
        uint32_t dt_valid:1;            /* Device Table valid. */
        uint32_t pt_valid:1;            /* Page translation valid. */
        uint16_t :7;                    /* Reserved[8:2] */
        uint8_t  pt_level:3;            /* Paging level, 0 to disable. */
        uint64_t pt_base:40;            /* Page table root pointer. */
        uint8_t  :3;                    /* Reserved[54:52] */
        uint8_t  gv_valid:1;            /* Revision 2, GVA to SPA. */
        uint8_t  gv_level:2;            /* Revision 2, GLX level. */
        uint8_t  gv_cr3_lsb:3;          /* Revision 2, GCR3[14:12] */
        uint8_t  read_allow:1;          /* I/O read enabled. */
        uint8_t  write_allow:1;         /* I/O write enabled. */
        uint8_t  :1;                    /* Reserved[63] */
        uint16_t domain_id:16;          /* Domain ID */
        uint16_t gv_cr3_lsb2:16;        /* Revision 2, GCR3[30:15] */
        uint8_t  iotlb_enable:1;        /* Device support IOTLB */
        uint8_t  sup_second_io_fault:1; /* Suppress subsequent I/O faults. */
        uint8_t  sup_all_io_fault:1;    /* Suppress all I/O page faults. */
        uint8_t  IOctl:2;               /* Port I/O control. */
        uint8_t  iotlb_cache_disable:1; /* IOTLB cache hints. */
        uint8_t  snoop_disable:1;       /* Snoop disable. */
        uint8_t  allow_ex:1;            /* Allow exclusion. */
        uint8_t  sysmgmt:2;             /* System management message.*/
        uint8_t  :1;                    /* Reserved[106] */
        uint32_t gv_cr3_msb:21;         /* Revision 2, GCR3[51:31] */
        uint8_t  intmap_valid:1;        /* Interrupt map valid. */
        uint8_t  intmap_len:4;          /* Interrupt map table length. */
        uint8_t  intmap_ign:1;          /* Ignore unmapped interrupts. */
        uint64_t intmap_base:46;        /* IntMap base. */
        uint8_t  :4;                    /* Reserved[183:180] */
        uint8_t  init_pass:1;           /* INIT pass through or PT */
        uint8_t  extintr_pass:1;        /* External Interrupt PT */
        uint8_t  nmi_pass:1;            /* NMI PT */
        uint8_t  :1;                    /* Reserved[187] */
        uint8_t  intr_ctrl:2;           /* Interrupt control */
        uint8_t  lint0_pass:1;          /* LINT0 PT */
        uint8_t  lint1_pass:1;          /* LINT1 PT */
        uint64_t :64;                   /* Reserved[255:192] */
} __attribute__((__packed__));
CTASSERT(sizeof(struct amdvi_dte) == 32);

/*
 * IOMMU command entry.
 */
struct amdvi_cmd {
        uint32_t        word0;
        uint32_t        word1:28;
        uint8_t         opcode:4;
        uint64_t        addr;
} __attribute__((__packed__));

/* Command opcodes. */
#define AMDVI_CMP_WAIT_OPCODE   0x1     /* Completion wait. */
#define AMDVI_INVD_DTE_OPCODE   0x2     /* Invalidate device table entry. */
#define AMDVI_INVD_PAGE_OPCODE  0x3     /* Invalidate pages. */
#define AMDVI_INVD_IOTLB_OPCODE 0x4     /* Invalidate IOTLB pages. */
#define AMDVI_INVD_INTR_OPCODE  0x5     /* Invalidate Interrupt table. */
#define AMDVI_PREFETCH_PAGES_OPCODE     0x6     /* Prefetch IOMMU pages. */
#define AMDVI_COMP_PPR_OPCODE   0x7     /* Complete PPR request. */
#define AMDVI_INV_ALL_OPCODE    0x8     /* Invalidate all. */

/* Completion wait attributes. */
#define AMDVI_CMP_WAIT_STORE    BIT(0)  /* Write back data. */
#define AMDVI_CMP_WAIT_INTR     BIT(1)  /* Completion wait interrupt. */
#define AMDVI_CMP_WAIT_FLUSH    BIT(2)  /* Flush queue. */

/* Invalidate page. */
#define AMDVI_INVD_PAGE_S       BIT(0)  /* Invalidation size. */
#define AMDVI_INVD_PAGE_PDE     BIT(1)  /* Invalidate PDE. */
#define AMDVI_INVD_PAGE_GN_GVA  BIT(2)  /* GPA or GVA. */

#define AMDVI_INVD_PAGE_ALL_ADDR        (0x7FFFFFFFFFFFFULL << 12)

/* Invalidate IOTLB. */
#define AMDVI_INVD_IOTLB_S      BIT(0)  /* Invalidation size 4k or addr */
#define AMDVI_INVD_IOTLB_GN_GVA BIT(2)  /* GPA or GVA. */

#define AMDVI_INVD_IOTLB_ALL_ADDR       (0x7FFFFFFFFFFFFULL << 12)
/* XXX: add more command entries. */

/*
 * IOMMU event entry.
 */
struct amdvi_event {
        uint16_t        devid;
        uint16_t        pasid_hi;
        uint16_t        pasid_domid;    /* PASID low or DomainID */
        uint16_t        flag:12;
        uint8_t         opcode:4;
        uint64_t        addr;
} __attribute__((__packed__));
CTASSERT(sizeof(struct amdvi_event) == 16);

/* Various event types. */
#define AMDVI_EVENT_INVALID_DTE         0x1
#define AMDVI_EVENT_PFAULT              0x2
#define AMDVI_EVENT_DTE_HW_ERROR        0x3
#define AMDVI_EVENT_PAGE_HW_ERROR       0x4
#define AMDVI_EVENT_ILLEGAL_CMD         0x5
#define AMDVI_EVENT_CMD_HW_ERROR        0x6
#define AMDVI_EVENT_IOTLB_TIMEOUT       0x7
#define AMDVI_EVENT_INVALID_DTE_REQ     0x8
#define AMDVI_EVENT_INVALID_PPR_REQ     0x9
#define AMDVI_EVENT_COUNTER_ZERO        0xA

#define AMDVI_EVENT_FLAG_MASK           0x1FF   /* Mask for event flags. */
#define AMDVI_EVENT_FLAG_TYPE(x)        (((x) >> 9) & 0x3)

/*
 * IOMMU control block.
 */
struct amdvi_ctrl {
        struct {
                uint16_t size:9;
                uint16_t :3;
                uint64_t base:40;       /* Devtable register base. */
                uint16_t :12;
        } dte;
        struct {
                uint16_t :12;
                uint64_t base:40;
                uint8_t  :4;
                uint8_t  len:4;
                uint8_t  :4;
        } cmd;
        struct {
                uint16_t :12;
                uint64_t base:40;
                uint8_t  :4;
                uint8_t  len:4;
                uint8_t  :4;
        } event;
        uint16_t control :13;
        uint64_t         :51;
        struct {
                uint8_t  enable:1;
                uint8_t  allow:1;
                uint16_t :10;
                uint64_t base:40;
                uint16_t :12;
                uint16_t :12;
                uint64_t limit:40;
                uint16_t :12;
        } excl;
        /*
         * Revision 2 only.
         */
        uint64_t ex_feature;
        struct {
                uint16_t :12;
                uint64_t base:40;
                uint8_t  :4;
                uint8_t  len:4;
                uint8_t  :4;
        } ppr;
        uint64_t first_event;
        uint64_t second_event;
        uint64_t event_status;
        /* Revision 2 only, end. */
        uint8_t  pad1[0x1FA8];          /* Padding. */
        uint32_t cmd_head:19;
        uint64_t :45;
        uint32_t cmd_tail:19;
        uint64_t :45;
        uint32_t evt_head:19;
        uint64_t :45;
        uint32_t evt_tail:19;
        uint64_t :45;
        uint32_t status:19;
        uint64_t :45;
        uint64_t pad2;
        uint8_t  :4;
        uint16_t ppr_head:15;
        uint64_t :45;
        uint8_t  :4;
        uint16_t ppr_tail:15;
        uint64_t :45;
        uint8_t  pad3[0x1FC0];          /* Padding. */

        /* XXX: More for rev2. */
} __attribute__((__packed__));
CTASSERT(offsetof(struct amdvi_ctrl, pad1)== 0x58);
CTASSERT(offsetof(struct amdvi_ctrl, pad2)== 0x2028);
CTASSERT(offsetof(struct amdvi_ctrl, pad3)== 0x2040);

#define AMDVI_MMIO_V1_SIZE      (4 * PAGE_SIZE) /* v1 size */
/*
 * AMF IOMMU v2 size including event counters
 */
#define AMDVI_MMIO_V2_SIZE      (8 * PAGE_SIZE)

CTASSERT(sizeof(struct amdvi_ctrl) == 0x4000);
CTASSERT(sizeof(struct amdvi_ctrl) == AMDVI_MMIO_V1_SIZE);

/* IVHD flag */
#define IVHD_FLAG_HTT           BIT(0)  /* Hypertransport Tunnel. */
#define IVHD_FLAG_PPW           BIT(1)  /* Pass posted write. */
#define IVHD_FLAG_RPPW          BIT(2)  /* Response pass posted write. */
#define IVHD_FLAG_ISOC          BIT(3)  /* Isoc support. */
#define IVHD_FLAG_IOTLB         BIT(4)  /* IOTLB support. */
#define IVHD_FLAG_COH           BIT(5)  /* Coherent control, default 1 */
#define IVHD_FLAG_PFS           BIT(6)  /* Prefetch IOMMU pages. */
#define IVHD_FLAG_PPRS          BIT(7)  /* Peripheral page support. */

/* IVHD device entry data setting. */
#define IVHD_DEV_LINT0_PASS     BIT(6)  /* LINT0 interrupts. */
#define IVHD_DEV_LINT1_PASS     BIT(7)  /* LINT1 interrupts. */

/* Bit[5:4] for System Mgmt. Bit3 is reserved. */
#define IVHD_DEV_INIT_PASS      BIT(0)  /* INIT */
#define IVHD_DEV_EXTINTR_PASS   BIT(1)  /* ExtInt */
#define IVHD_DEV_NMI_PASS       BIT(2)  /* NMI */

/* IVHD 8-byte extended data settings. */
#define IVHD_DEV_EXT_ATS_DISABLE        BIT(31) /* Disable ATS */

/* IOMMU control register. */
#define AMDVI_CTRL_EN           BIT(0)  /* IOMMU enable. */
#define AMDVI_CTRL_HTT          BIT(1)  /* Hypertransport tunnel enable. */
#define AMDVI_CTRL_ELOG         BIT(2)  /* Event log enable. */
#define AMDVI_CTRL_ELOGINT      BIT(3)  /* Event log interrupt. */
#define AMDVI_CTRL_COMINT       BIT(4)  /* Completion wait interrupt. */
#define AMDVI_CTRL_PPW          BIT(8)
#define AMDVI_CTRL_RPPW         BIT(9)
#define AMDVI_CTRL_COH          BIT(10)
#define AMDVI_CTRL_ISOC         BIT(11)
#define AMDVI_CTRL_CMD          BIT(12) /* Command buffer enable. */
#define AMDVI_CTRL_PPRLOG       BIT(13)
#define AMDVI_CTRL_PPRINT       BIT(14)
#define AMDVI_CTRL_PPREN        BIT(15)
#define AMDVI_CTRL_GTE          BIT(16) /* Guest translation enable. */
#define AMDVI_CTRL_GAE          BIT(17) /* Guest APIC enable. */

/* Invalidation timeout. */
#define AMDVI_CTRL_INV_NO_TO    0       /* No timeout. */
#define AMDVI_CTRL_INV_TO_1ms   1       /* 1 ms */
#define AMDVI_CTRL_INV_TO_10ms  2       /* 10 ms */
#define AMDVI_CTRL_INV_TO_100ms 3       /* 100 ms */
#define AMDVI_CTRL_INV_TO_1S    4       /* 1 second */
#define AMDVI_CTRL_INV_TO_10S   5       /* 10 second */
#define AMDVI_CTRL_INV_TO_100S  6       /* 100 second */

/*
 * Max number of PCI devices.
 * 256 bus x 32 slot/devices x 8 functions.
 */
#define PCI_NUM_DEV_MAX         0x10000

/* Maximum number of domains supported by IOMMU. */
#define AMDVI_MAX_DOMAIN        (BIT(16) - 1)

/*
 * IOMMU Page Table attributes.
 */
#define AMDVI_PT_PRESENT        BIT(0)
#define AMDVI_PT_COHERENT       BIT(60)
#define AMDVI_PT_READ           BIT(61)
#define AMDVI_PT_WRITE          BIT(62)

#define AMDVI_PT_RW             (AMDVI_PT_READ | AMDVI_PT_WRITE)
#define AMDVI_PT_MASK           0xFFFFFFFFFF000UL /* Only [51:12] for PA */

#define AMDVI_PD_LEVEL_SHIFT    9
#define AMDVI_PD_SUPER(x)       (((x) >> AMDVI_PD_LEVEL_SHIFT) == 7)
/*
 * IOMMU Status, offset 0x2020
 */
#define AMDVI_STATUS_EV_OF              BIT(0)  /* Event overflow. */
#define AMDVI_STATUS_EV_INTR            BIT(1)  /* Event interrupt. */
/* Completion wait command completed. */
#define AMDVI_STATUS_CMP                BIT(2)

#define IVRS_CTRL_RID                   1       /* MMIO RID */

/* ACPI IVHD */
struct ivhd_dev_cfg {
        uint32_t start_id;
        uint32_t end_id;
        uint8_t  data;                  /* Device configuration. */
        bool     enable_ats;            /* ATS enabled for the device. */
        int      ats_qlen;              /* ATS invalidation queue depth. */
};

struct amdvi_domain {
        uint64_t *ptp;                  /* Highest level page table */
        int     ptp_level;              /* Level of page tables */
        u_int   id;                     /* Domain id */
        SLIST_ENTRY (amdvi_domain) next;
};

/*
 * Different type of IVHD.
 * XXX: Use AcpiIvrsType once new IVHD types are available.
*/
enum IvrsType
{
        IVRS_TYPE_HARDWARE_LEGACY = ACPI_IVRS_TYPE_HARDWARE1,
                                        /* Legacy without EFRi support. */
        IVRS_TYPE_HARDWARE_EFR    = ACPI_IVRS_TYPE_HARDWARE2,
                                                /* With EFR support. */
        IVRS_TYPE_HARDWARE_MIXED  = 0x40, /* Mixed with EFR support. */
};

/*
 * AMD IOMMU softc.
 */
struct amdvi_softc {
        struct amdvi_ctrl *ctrl;        /* Control area. */
        device_t        dev;            /* IOMMU device. */
        device_t        pci_dev;        /* IOMMU PCI function device. */
        enum IvrsType   ivhd_type;      /* IOMMU IVHD type. */
        bool            iotlb;          /* IOTLB supported by IOMMU */
        struct amdvi_cmd *cmd;          /* Command descriptor area. */
        int             cmd_max;        /* Max number of commands. */
        uint64_t        cmp_data;       /* Command completion write back. */
        struct amdvi_event *event;      /* Event descriptor area. */
        int             event_max;      /* Max number of events. */
        /* ACPI various flags. */
        uint32_t        ivhd_flag;      /* ACPI IVHD flag. */
        uint32_t        ivhd_feature;   /* ACPI v1 Reserved or v2 attribute. */
        uint64_t        ext_feature;    /* IVHD EFR */
        /* PCI related. */
        uint16_t        cap_off;        /* PCI Capability offset. */
        uint8_t         pci_cap;        /* PCI capability. */
        uint16_t        pci_seg;        /* IOMMU PCI domain/segment. */
        uint16_t        pci_rid;        /* PCI BDF of IOMMU */

        /* ACPI device configuration for end points. */
        struct          ivhd_dev_cfg *dev_cfg;
        int             dev_cfg_cnt;
        int             dev_cfg_cap;

        /* Software statistics. */
        uint64_t        event_intr_cnt; /* Total event INTR count. */
        uint64_t        total_cmd;      /* Total number of commands. */
};

int     amdvi_setup_hw(struct amdvi_softc *softc);
int     amdvi_teardown_hw(struct amdvi_softc *softc);
#endif /* _AMDVI_PRIV_H_ */