#include <drm/amdxdna_accel.h>
#include <drm/drm_managed.h>
#include <drm/drm_print.h>
#include <linux/firmware.h>
#include <linux/sizes.h>
#include "aie.h"
#include "aie4_msg_priv.h"
#include "aie4_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_mailbox_helper.h"
#include "amdxdna_pci_drv.h"
#define NO_IOHUB 0
#define PSP_NOTIFY_INTR 0xD007BE11
#define AIE4_TOTAL_COLUMN 3
struct mailbox_info {
__u32 valid;
__u32 protocol_major;
__u32 protocol_minor;
__u32 x2i_tail_offset;
__u32 x2i_head_offset;
__u32 x2i_buffer_addr;
__u32 x2i_buffer_size;
__u32 i2x_tail_offset;
__u32 i2x_head_offset;
__u32 i2x_buffer_addr;
__u32 i2x_buffer_size;
__u32 i2x_msi_idx;
__u32 reserved[4];
};
static int aie4_fw_is_alive(struct amdxdna_dev *xdna)
{
const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
u32 __iomem *src;
u32 fw_is_valid;
int ret;
src = ndev->rbuf_base + npriv->mbox_info_off;
ret = readx_poll_timeout(readl, src + offsetof(struct mailbox_info, valid),
fw_is_valid, (fw_is_valid == 0x1),
AIE_INTERVAL, AIE_TIMEOUT);
if (ret)
XDNA_ERR(xdna, "fw_is_valid=%d after %d ms",
fw_is_valid, DIV_ROUND_CLOSEST(AIE_TIMEOUT, 1000000));
return ret;
}
static void aie4_read_mbox_info(struct amdxdna_dev *xdna,
struct mailbox_info *mbox_info)
{
const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
u32 *dst = (u32 *)mbox_info;
u32 __iomem *src;
int i;
src = ndev->rbuf_base + npriv->mbox_info_off;
for (i = 0; i < sizeof(*mbox_info) / sizeof(u32); i++)
dst[i] = readl(&src[i]);
}
static int aie4_mailbox_info(struct amdxdna_dev *xdna,
struct mailbox_info *mbox_info)
{
int ret;
ret = aie4_fw_is_alive(xdna);
if (ret)
return ret;
aie4_read_mbox_info(xdna, mbox_info);
ret = aie_check_protocol(&xdna->dev_handle->aie,
mbox_info->protocol_major,
mbox_info->protocol_minor);
if (ret)
XDNA_ERR(xdna, "mailbox major.minor %d.%d is not supported",
mbox_info->protocol_major, mbox_info->protocol_minor);
return ret;
}
static void aie4_mailbox_fini(struct amdxdna_dev_hdl *ndev)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
aie_destroy_chann(&ndev->aie, &ndev->aie.mgmt_chann);
drmm_kfree(&xdna->ddev, ndev->mbox);
ndev->mbox = NULL;
}
static int aie4_irq_init(struct amdxdna_dev *xdna)
{
struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
int ret, nvec;
nvec = pci_msix_vec_count(pdev);
XDNA_DBG(xdna, "irq vectors:%d", nvec);
if (nvec <= 0) {
XDNA_ERR(xdna, "does not get number of interrupt vector");
return -EINVAL;
}
ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
if (ret < 0) {
XDNA_ERR(xdna, "failed to alloc irq vector, ret: %d", ret);
return ret;
}
return 0;
}
static int aie4_mailbox_start(struct amdxdna_dev *xdna,
struct mailbox_info *mbi)
{
struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
struct xdna_mailbox_chann_res *i2x;
struct xdna_mailbox_chann_res *x2i;
int mgmt_mb_irq;
int ret;
struct xdna_mailbox_res mbox_res = {
.ringbuf_base = ndev->rbuf_base,
.ringbuf_size = pci_resource_len(pdev, npriv->mbox_rbuf_bar),
.mbox_base = ndev->mbox_base,
.mbox_size = pci_resource_len(pdev, npriv->mbox_bar),
.name = "xdna_aie4_mailbox",
};
i2x = &ndev->aie.mgmt_i2x;
x2i = &ndev->aie.mgmt_x2i;
x2i->mb_head_ptr_reg = mbi->x2i_head_offset;
x2i->mb_tail_ptr_reg = mbi->x2i_tail_offset;
x2i->rb_start_addr = mbi->x2i_buffer_addr;
x2i->rb_size = mbi->x2i_buffer_size;
i2x->rb_start_addr = mbi->i2x_buffer_addr;
i2x->rb_size = mbi->i2x_buffer_size;
i2x->mb_head_ptr_reg = mbi->i2x_head_offset;
i2x->mb_tail_ptr_reg = mbi->i2x_tail_offset;
ndev->aie.mgmt_chan_idx = mbi->i2x_msi_idx;
aie_dump_mgmt_chann_debug(&ndev->aie);
ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res);
if (!ndev->mbox) {
XDNA_ERR(xdna, "failed to create mailbox device");
return -ENODEV;
}
ndev->aie.mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox);
if (!ndev->aie.mgmt_chann) {
XDNA_ERR(xdna, "failed to alloc mailbox channel");
return -ENODEV;
}
mgmt_mb_irq = pci_irq_vector(pdev, ndev->aie.mgmt_chan_idx);
if (mgmt_mb_irq < 0) {
XDNA_ERR(xdna, "failed to alloc irq vector, return %d", mgmt_mb_irq);
ret = mgmt_mb_irq;
goto free_channel;
}
ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann,
&ndev->aie.mgmt_x2i,
&ndev->aie.mgmt_i2x,
NO_IOHUB,
mgmt_mb_irq);
if (ret) {
XDNA_ERR(xdna, "failed to start management mailbox channel");
ret = -EINVAL;
goto free_channel;
}
XDNA_DBG(xdna, "Mailbox management channel created");
return 0;
free_channel:
xdna_mailbox_free_channel(ndev->aie.mgmt_chann);
ndev->aie.mgmt_chann = NULL;
return ret;
}
static int aie4_mailbox_init(struct amdxdna_dev_hdl *ndev)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
struct mailbox_info mbox_info;
int ret;
ret = aie4_mailbox_info(xdna, &mbox_info);
if (ret)
return ret;
return aie4_mailbox_start(xdna, &mbox_info);
}
static void aie4_fw_stop(struct amdxdna_dev_hdl *ndev)
{
aie_psp_stop(ndev->aie.psp_hdl);
aie_smu_fini(ndev->aie.smu_hdl);
}
static int aie4_fw_start(struct amdxdna_dev_hdl *ndev)
{
int ret;
ret = aie_smu_init(ndev->aie.smu_hdl);
if (ret) {
XDNA_ERR(ndev->aie.xdna, "failed to init smu, ret %d", ret);
return ret;
}
ret = aie_psp_start(ndev->aie.psp_hdl);
if (ret) {
XDNA_ERR(ndev->aie.xdna, "failed to start psp, ret %d", ret);
aie_smu_fini(ndev->aie.smu_hdl);
}
return ret;
}
static int aie4_partition_init(struct amdxdna_dev_hdl *ndev)
{
DECLARE_AIE_MSG(aie4_msg_create_partition, AIE4_MSG_OP_CREATE_PARTITION);
struct amdxdna_dev *xdna = ndev->aie.xdna;
int ret;
req.partition_col_start = 0;
req.partition_col_count = AIE4_TOTAL_COLUMN;
ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
if (ret) {
XDNA_ERR(xdna, "partition init failed: %d", ret);
return ret;
}
ndev->partition_id = resp.partition_id;
return 0;
}
static void aie4_partition_fini(struct amdxdna_dev_hdl *ndev)
{
DECLARE_AIE_MSG(aie4_msg_destroy_partition, AIE4_MSG_OP_DESTROY_PARTITION);
struct amdxdna_dev *xdna = ndev->aie.xdna;
int ret;
req.partition_id = ndev->partition_id;
ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
if (ret)
XDNA_ERR(xdna, "partition fini failed: %d", ret);
}
static int aie4_query(struct amdxdna_dev_hdl *ndev)
{
return aie4_query_aie_metadata(ndev, &ndev->aie.metadata);
}
static int aie4_pf_hw_start(struct amdxdna_dev_hdl *ndev)
{
int ret;
ret = aie4_fw_start(ndev);
if (ret)
return ret;
ret = aie4_mailbox_init(ndev);
if (ret)
goto stop_fw;
ret = aie4_attach_work_buffer(ndev);
if (ret)
goto mbox_fini;
return 0;
mbox_fini:
aie4_mailbox_fini(ndev);
stop_fw:
aie4_fw_stop(ndev);
return ret;
}
static void aie4_pf_hw_stop(struct amdxdna_dev_hdl *ndev)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
aie4_suspend_fw(ndev);
aie4_mailbox_fini(ndev);
aie4_fw_stop(ndev);
}
static int aie4_vf_hw_start(struct amdxdna_dev_hdl *ndev)
{
int ret;
ret = aie4_mailbox_init(ndev);
if (ret)
return ret;
ret = aie4_query(ndev);
if (ret)
goto mailbox_fini;
ret = aie4_partition_init(ndev);
if (ret)
goto mailbox_fini;
return 0;
mailbox_fini:
aie4_mailbox_fini(ndev);
return ret;
}
static void aie4_vf_hw_stop(struct amdxdna_dev_hdl *ndev)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
aie4_partition_fini(ndev);
aie4_mailbox_fini(ndev);
}
static int aie4_request_firmware(struct amdxdna_dev_hdl *ndev,
const struct firmware **npufw,
const struct firmware **certfw)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
char fw_name[128];
int ret;
ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s",
pdev->device, pdev->revision, ndev->priv->npufw_path);
if (ret >= sizeof(fw_name)) {
XDNA_ERR(xdna, "npu firmware path is truncated");
return -EINVAL;
}
ret = request_firmware(npufw, fw_name, &pdev->dev);
if (ret) {
XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret);
return ret;
}
ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s",
pdev->device, pdev->revision, ndev->priv->certfw_path);
if (ret >= sizeof(fw_name)) {
XDNA_ERR(xdna, "cert firmware path is truncated");
ret = -EINVAL;
goto release_npufw;
}
ret = request_firmware(certfw, fw_name, &pdev->dev);
if (ret) {
XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret);
goto release_npufw;
}
return 0;
release_npufw:
release_firmware(*npufw);
return ret;
}
static void aie4_release_firmware(struct amdxdna_dev_hdl *ndev,
const struct firmware *npufw,
const struct firmware *certfw)
{
release_firmware(certfw);
release_firmware(npufw);
}
static int aie4_prepare_firmware(struct amdxdna_dev_hdl *ndev,
const struct firmware *npufw,
const struct firmware *certfw,
void __iomem *tbl[PCI_NUM_RESOURCES])
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
struct psp_config psp_conf;
struct smu_config smu_conf;
int i;
psp_conf.fw_size = npufw->size;
psp_conf.fw_buf = npufw->data;
psp_conf.certfw_size = certfw->size;
psp_conf.certfw_buf = certfw->data;
psp_conf.arg2_mask = ~0;
psp_conf.notify_val = PSP_NOTIFY_INTR;
for (i = 0; i < PSP_MAX_REGS; i++)
psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
ndev->aie.psp_hdl = aiem_psp_create(&xdna->ddev, &psp_conf);
if (!ndev->aie.psp_hdl) {
XDNA_ERR(xdna, "failed to create psp");
return -ENOMEM;
}
for (i = 0; i < SMU_MAX_REGS; i++)
smu_conf.smu_regs[i] = tbl[SMU_REG_BAR(ndev, i)] + SMU_REG_OFF(ndev, i);
ndev->aie.smu_hdl = aiem_smu_create(&xdna->ddev, &smu_conf);
if (!ndev->aie.smu_hdl) {
XDNA_ERR(xdna, "failed to create smu");
return -ENOMEM;
}
return 0;
}
static int aie4_load_fw(struct amdxdna_dev_hdl *ndev,
void __iomem *tbl[PCI_NUM_RESOURCES])
{
const struct firmware *npufw, *certfw;
int ret;
if (!ndev->priv->npufw_path && !ndev->priv->certfw_path)
return 0;
ret = aie4_request_firmware(ndev, &npufw, &certfw);
if (ret)
return ret;
ret = aie4_prepare_firmware(ndev, npufw, certfw, tbl);
aie4_release_firmware(ndev, npufw, certfw);
return ret;
}
static int aie4m_pcidev_init(struct amdxdna_dev *xdna)
{
struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
struct amdxdna_dev_hdl *ndev;
void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
unsigned long bars = 0;
int ret, i;
ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
if (!ndev)
return -ENOMEM;
ndev->priv = xdna->dev_info->dev_priv;
ndev->aie.xdna = xdna;
xdna->dev_handle = ndev;
xa_init_flags(&ndev->cert_comp_xa, XA_FLAGS_ALLOC);
mutex_init(&ndev->cert_comp_lock);
ret = pcim_enable_device(pdev);
if (ret) {
XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret);
return ret;
}
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
if (ret) {
XDNA_ERR(xdna, "failed to set DMA mask to 64:%d", ret);
return ret;
}
for (i = 0; i < PSP_MAX_REGS; i++)
set_bit(PSP_REG_BAR(ndev, i), &bars);
for (i = 0; i < SMU_MAX_REGS; i++)
set_bit(SMU_REG_BAR(ndev, i), &bars);
set_bit(xdna->dev_info->mbox_bar, &bars);
set_bit(xdna->dev_info->sram_bar, &bars);
for (i = 0; i < PCI_NUM_RESOURCES; i++) {
if (!test_bit(i, &bars))
continue;
tbl[i] = pcim_iomap(pdev, i, 0);
if (!tbl[i]) {
XDNA_ERR(xdna, "map bar %d failed", i);
return -ENOMEM;
}
}
ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
ndev->rbuf_base = tbl[xdna->dev_info->sram_bar];
pci_set_master(pdev);
ret = aie4_load_fw(ndev, tbl);
if (ret)
return ret;
ret = aie4_irq_init(xdna);
if (ret)
return ret;
amdxdna_vbnv_init(xdna);
XDNA_DBG(xdna, "init finished");
return 0;
}
static int aie4_doorbell_mmap(struct amdxdna_client *client, struct vm_area_struct *vma)
{
struct amdxdna_dev *xdna = client->xdna;
struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
phys_addr_t res_start;
unsigned long pfn;
int ret;
if (!aie4_hwctx_valid_doorbell(client, vma->vm_pgoff)) {
XDNA_ERR(xdna, "Invalid doorbell page offset 0x%lx", vma->vm_pgoff);
return -EINVAL;
}
if (vma_pages(vma) != 1) {
XDNA_ERR(xdna, "can only map one page, got %ld", vma_pages(vma));
return -EINVAL;
}
res_start = pci_resource_start(pdev, xdna->dev_info->doorbell_bar) + npriv->doorbell_off;
pfn = PHYS_PFN(res_start) + vma->vm_pgoff;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
vm_flags_set(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP);
ret = io_remap_pfn_range(vma, vma->vm_start,
pfn,
PAGE_SIZE,
vma->vm_page_prot);
XDNA_DBG(xdna, "doorbell ret %d", ret);
return ret;
}
static int aie4_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
{
struct amdxdna_dev *xdna = client->xdna;
struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
int ret;
switch (args->param) {
case DRM_AMDXDNA_QUERY_AIE_METADATA:
ret = amdxdna_get_metadata(&ndev->aie, client, args);
break;
default:
XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
ret = -EOPNOTSUPP;
}
XDNA_DBG(xdna, "Got param %d", args->param);
return ret;
}
static int aie4_alloc_work_buffer(struct amdxdna_dev_hdl *ndev)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
u32 buf_size = AIE4_WORK_BUFFER_MIN_SIZE;
ndev->work_buf = amdxdna_alloc_msg_buffer(xdna, &buf_size,
&ndev->work_buf_addr);
if (IS_ERR(ndev->work_buf)) {
int ret = PTR_ERR(ndev->work_buf);
XDNA_ERR(xdna, "Failed to alloc work buffer, size 0x%x",
AIE4_WORK_BUFFER_MIN_SIZE);
ndev->work_buf = NULL;
return ret;
}
ndev->work_buf_size = buf_size;
XDNA_DBG(xdna, "Work buffer allocated: size 0x%x", buf_size);
return 0;
}
static void aie4_free_work_buffer(struct amdxdna_dev_hdl *ndev)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
if (!ndev->work_buf)
return;
amdxdna_free_msg_buffer(xdna, ndev->work_buf_size, ndev->work_buf,
ndev->work_buf_addr);
ndev->work_buf = NULL;
}
static int aie4_pf_init(struct amdxdna_dev *xdna)
{
int ret;
ret = aie4m_pcidev_init(xdna);
if (ret)
return ret;
ret = aie4_alloc_work_buffer(xdna->dev_handle);
if (ret)
return ret;
ret = aie4_pf_hw_start(xdna->dev_handle);
if (ret)
goto free_work_buf;
return 0;
free_work_buf:
aie4_free_work_buffer(xdna->dev_handle);
return ret;
}
static int aie4_vf_init(struct amdxdna_dev *xdna)
{
int ret;
ret = aie4m_pcidev_init(xdna);
if (ret)
return ret;
return aie4_vf_hw_start(xdna->dev_handle);
}
static void aie4_pf_fini(struct amdxdna_dev *xdna)
{
aie4_sriov_stop(xdna->dev_handle);
aie4_pf_hw_stop(xdna->dev_handle);
aie4_free_work_buffer(xdna->dev_handle);
}
static void aie4_vf_fini(struct amdxdna_dev *xdna)
{
aie4_vf_hw_stop(xdna->dev_handle);
}
const struct amdxdna_dev_ops aie4_pf_ops = {
.init = aie4_pf_init,
.fini = aie4_pf_fini,
.sriov_configure = aie4_sriov_configure,
};
const struct amdxdna_dev_ops aie4_vf_ops = {
.init = aie4_vf_init,
.fini = aie4_vf_fini,
.hwctx_init = aie4_hwctx_init,
.hwctx_fini = aie4_hwctx_fini,
.mmap = aie4_doorbell_mmap,
.cmd_wait = aie4_cmd_wait,
.get_aie_info = aie4_get_info,
};
