#include <linux/module.h>
#include <linux/auxiliary_bus.h>
#include <linux/pci.h>
#include <linux/vmalloc.h>
#include <linux/bitfield.h>
#include <linux/string.h>
#include <uapi/fwctl/fwctl.h>
#include <uapi/fwctl/pds.h>
#include <linux/fwctl.h>
#include <linux/pds/pds_common.h>
#include <linux/pds/pds_core_if.h>
#include <linux/pds/pds_adminq.h>
#include <linux/pds/pds_auxbus.h>
struct pdsfc_uctx {
struct fwctl_uctx uctx;
u32 uctx_caps;
};
struct pdsfc_rpc_endpoint_info {
u32 endpoint;
dma_addr_t operations_pa;
struct pds_fwctl_query_data *operations;
struct mutex lock;
};
struct pdsfc_dev {
struct fwctl_device fwctl;
struct pds_auxiliary_dev *padev;
u32 caps;
struct pds_fwctl_ident ident;
dma_addr_t endpoints_pa;
struct pds_fwctl_query_data *endpoints;
struct pdsfc_rpc_endpoint_info *endpoint_info;
};
static int pdsfc_open_uctx(struct fwctl_uctx *uctx)
{
struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl);
struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx);
pdsfc_uctx->uctx_caps = pdsfc->caps;
return 0;
}
static void pdsfc_close_uctx(struct fwctl_uctx *uctx)
{
}
static void *pdsfc_info(struct fwctl_uctx *uctx, size_t *length)
{
struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx);
struct fwctl_info_pds *info;
info = kzalloc_obj(*info);
if (!info)
return ERR_PTR(-ENOMEM);
info->uctx_caps = pdsfc_uctx->uctx_caps;
return info;
}
static int pdsfc_identify(struct pdsfc_dev *pdsfc)
{
struct device *dev = &pdsfc->fwctl.dev;
union pds_core_adminq_comp comp = {0};
union pds_core_adminq_cmd cmd;
struct pds_fwctl_ident *ident;
dma_addr_t ident_pa;
int err;
ident = dma_alloc_coherent(dev->parent, sizeof(*ident), &ident_pa, GFP_KERNEL);
if (!ident) {
dev_err(dev, "Failed to map ident buffer\n");
return -ENOMEM;
}
cmd = (union pds_core_adminq_cmd) {
.fwctl_ident = {
.opcode = PDS_FWCTL_CMD_IDENT,
.version = 0,
.len = cpu_to_le32(sizeof(*ident)),
.ident_pa = cpu_to_le64(ident_pa),
}
};
err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
if (err)
dev_err(dev, "Failed to send adminq cmd opcode: %u err: %d\n",
cmd.fwctl_ident.opcode, err);
else
pdsfc->ident = *ident;
dma_free_coherent(dev->parent, sizeof(*ident), ident, ident_pa);
return err;
}
static void pdsfc_free_endpoints(struct pdsfc_dev *pdsfc)
{
struct device *dev = &pdsfc->fwctl.dev;
u32 num_endpoints;
int i;
if (!pdsfc->endpoints)
return;
num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries);
for (i = 0; pdsfc->endpoint_info && i < num_endpoints; i++)
mutex_destroy(&pdsfc->endpoint_info[i].lock);
vfree(pdsfc->endpoint_info);
pdsfc->endpoint_info = NULL;
dma_free_coherent(dev->parent, PAGE_SIZE,
pdsfc->endpoints, pdsfc->endpoints_pa);
pdsfc->endpoints = NULL;
pdsfc->endpoints_pa = DMA_MAPPING_ERROR;
}
static void pdsfc_free_operations(struct pdsfc_dev *pdsfc)
{
struct device *dev = &pdsfc->fwctl.dev;
u32 num_endpoints;
int i;
num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries);
for (i = 0; i < num_endpoints; i++) {
struct pdsfc_rpc_endpoint_info *ei = &pdsfc->endpoint_info[i];
if (ei->operations) {
dma_free_coherent(dev->parent, PAGE_SIZE,
ei->operations, ei->operations_pa);
ei->operations = NULL;
ei->operations_pa = DMA_MAPPING_ERROR;
}
}
}
static struct pds_fwctl_query_data *pdsfc_get_endpoints(struct pdsfc_dev *pdsfc,
dma_addr_t *pa)
{
struct device *dev = &pdsfc->fwctl.dev;
union pds_core_adminq_comp comp = {0};
struct pds_fwctl_query_data *data;
union pds_core_adminq_cmd cmd;
dma_addr_t data_pa;
int err;
data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL);
if (!data) {
dev_err(dev, "Failed to map endpoint list\n");
return ERR_PTR(-ENOMEM);
}
cmd = (union pds_core_adminq_cmd) {
.fwctl_query = {
.opcode = PDS_FWCTL_CMD_QUERY,
.entity = PDS_FWCTL_RPC_ROOT,
.version = 0,
.query_data_buf_len = cpu_to_le32(PAGE_SIZE),
.query_data_buf_pa = cpu_to_le64(data_pa),
}
};
err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
if (err) {
dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n",
cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err);
dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa);
return ERR_PTR(err);
}
*pa = data_pa;
return data;
}
static int pdsfc_init_endpoints(struct pdsfc_dev *pdsfc)
{
struct pds_fwctl_query_data_endpoint *ep_entry;
u32 num_endpoints;
int i;
pdsfc->endpoints = pdsfc_get_endpoints(pdsfc, &pdsfc->endpoints_pa);
if (IS_ERR(pdsfc->endpoints))
return PTR_ERR(pdsfc->endpoints);
num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries);
pdsfc->endpoint_info = vcalloc(num_endpoints,
sizeof(*pdsfc->endpoint_info));
if (!pdsfc->endpoint_info) {
pdsfc_free_endpoints(pdsfc);
return -ENOMEM;
}
ep_entry = (struct pds_fwctl_query_data_endpoint *)pdsfc->endpoints->entries;
for (i = 0; i < num_endpoints; i++) {
mutex_init(&pdsfc->endpoint_info[i].lock);
pdsfc->endpoint_info[i].endpoint = le32_to_cpu(ep_entry[i].id);
}
return 0;
}
static struct pds_fwctl_query_data *pdsfc_get_operations(struct pdsfc_dev *pdsfc,
dma_addr_t *pa, u32 ep)
{
struct pds_fwctl_query_data_operation *entries;
struct device *dev = &pdsfc->fwctl.dev;
union pds_core_adminq_comp comp = {0};
struct pds_fwctl_query_data *data;
union pds_core_adminq_cmd cmd;
dma_addr_t data_pa;
u32 num_entries;
int err;
int i;
data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL);
if (!data) {
dev_err(dev, "Failed to map operations list\n");
return ERR_PTR(-ENOMEM);
}
cmd = (union pds_core_adminq_cmd) {
.fwctl_query = {
.opcode = PDS_FWCTL_CMD_QUERY,
.entity = PDS_FWCTL_RPC_ENDPOINT,
.version = 0,
.query_data_buf_len = cpu_to_le32(PAGE_SIZE),
.query_data_buf_pa = cpu_to_le64(data_pa),
.ep = cpu_to_le32(ep),
}
};
err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
if (err) {
dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n",
cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err);
dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa);
return ERR_PTR(err);
}
*pa = data_pa;
entries = (struct pds_fwctl_query_data_operation *)data->entries;
num_entries = le32_to_cpu(data->num_entries);
dev_dbg(dev, "num_entries %d\n", num_entries);
for (i = 0; i < num_entries; i++) {
switch (entries[i].scope) {
case PDSFC_FW_CMD_ATTR_READ:
case PDSFC_FW_CMD_ATTR_WRITE:
case PDSFC_FW_CMD_ATTR_SYNC:
entries[i].scope = FWCTL_RPC_CONFIGURATION;
break;
case PDSFC_FW_CMD_ATTR_DEBUG_READ:
entries[i].scope = FWCTL_RPC_DEBUG_READ_ONLY;
break;
case PDSFC_FW_CMD_ATTR_DEBUG_WRITE:
entries[i].scope = FWCTL_RPC_DEBUG_WRITE;
break;
default:
entries[i].scope = FWCTL_RPC_DEBUG_WRITE_FULL;
break;
}
dev_dbg(dev, "endpoint %d operation: id %x scope %d\n",
ep, le32_to_cpu(entries[i].id), entries[i].scope);
}
return data;
}
static int pdsfc_validate_rpc(struct pdsfc_dev *pdsfc,
struct fwctl_rpc_pds *rpc,
enum fwctl_rpc_scope scope)
{
struct pds_fwctl_query_data_operation *op_entry;
struct pdsfc_rpc_endpoint_info *ep_info = NULL;
struct device *dev = &pdsfc->fwctl.dev;
u32 num_entries;
int i;
if (rpc->in.len > le32_to_cpu(pdsfc->ident.max_req_sz)) {
dev_dbg(dev, "Invalid request size %u, max %u\n",
rpc->in.len, le32_to_cpu(pdsfc->ident.max_req_sz));
return -EINVAL;
}
if (rpc->out.len > le32_to_cpu(pdsfc->ident.max_resp_sz)) {
dev_dbg(dev, "Invalid response size %u, max %u\n",
rpc->out.len, le32_to_cpu(pdsfc->ident.max_resp_sz));
return -EINVAL;
}
num_entries = le32_to_cpu(pdsfc->endpoints->num_entries);
for (i = 0; i < num_entries; i++) {
if (pdsfc->endpoint_info[i].endpoint == rpc->in.ep) {
ep_info = &pdsfc->endpoint_info[i];
break;
}
}
if (!ep_info) {
dev_dbg(dev, "Invalid endpoint %d\n", rpc->in.ep);
return -EINVAL;
}
mutex_lock(&ep_info->lock);
if (!ep_info->operations) {
struct pds_fwctl_query_data *operations;
operations = pdsfc_get_operations(pdsfc,
&ep_info->operations_pa,
rpc->in.ep);
if (IS_ERR(operations)) {
mutex_unlock(&ep_info->lock);
return -ENOMEM;
}
ep_info->operations = operations;
}
mutex_unlock(&ep_info->lock);
op_entry = (struct pds_fwctl_query_data_operation *)ep_info->operations->entries;
num_entries = le32_to_cpu(ep_info->operations->num_entries);
for (i = 0; i < num_entries; i++) {
if (PDS_FWCTL_RPC_OPCODE_CMP(rpc->in.op, le32_to_cpu(op_entry[i].id))) {
if (scope < op_entry[i].scope)
return -EPERM;
return 0;
}
}
dev_dbg(dev, "Invalid operation %d for endpoint %d\n", rpc->in.op, rpc->in.ep);
return -EINVAL;
}
static void *pdsfc_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
void *in, size_t in_len, size_t *out_len)
{
struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl);
struct device *dev = &uctx->fwctl->dev;
union pds_core_adminq_comp comp = {0};
dma_addr_t out_payload_dma_addr = 0;
dma_addr_t in_payload_dma_addr = 0;
struct fwctl_rpc_pds *rpc = in;
union pds_core_adminq_cmd cmd;
void *out_payload = NULL;
void *in_payload = NULL;
void *out = NULL;
int err;
err = pdsfc_validate_rpc(pdsfc, rpc, scope);
if (err)
return ERR_PTR(err);
if (rpc->in.len > 0) {
in_payload = memdup_user(u64_to_user_ptr(rpc->in.payload), rpc->in.len);
if (IS_ERR(in_payload)) {
dev_dbg(dev, "Failed to copy in_payload from user\n");
return in_payload;
}
in_payload_dma_addr = dma_map_single(dev->parent, in_payload,
rpc->in.len, DMA_TO_DEVICE);
err = dma_mapping_error(dev->parent, in_payload_dma_addr);
if (err) {
dev_dbg(dev, "Failed to map in_payload\n");
goto err_in_payload;
}
}
if (rpc->out.len > 0) {
out_payload = kzalloc(rpc->out.len, GFP_KERNEL);
if (!out_payload) {
dev_dbg(dev, "Failed to allocate out_payload\n");
err = -ENOMEM;
goto err_out_payload;
}
out_payload_dma_addr = dma_map_single(dev->parent, out_payload,
rpc->out.len, DMA_FROM_DEVICE);
err = dma_mapping_error(dev->parent, out_payload_dma_addr);
if (err) {
dev_dbg(dev, "Failed to map out_payload\n");
goto err_out_payload;
}
}
cmd = (union pds_core_adminq_cmd) {
.fwctl_rpc = {
.opcode = PDS_FWCTL_CMD_RPC,
.flags = cpu_to_le16(PDS_FWCTL_RPC_IND_REQ | PDS_FWCTL_RPC_IND_RESP),
.ep = cpu_to_le32(rpc->in.ep),
.op = cpu_to_le32(rpc->in.op),
.req_pa = cpu_to_le64(in_payload_dma_addr),
.req_sz = cpu_to_le32(rpc->in.len),
.resp_pa = cpu_to_le64(out_payload_dma_addr),
.resp_sz = cpu_to_le32(rpc->out.len),
}
};
err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
if (err) {
dev_dbg(dev, "%s: ep %d op %x req_pa %llx req_sz %d req_sg %d resp_pa %llx resp_sz %d resp_sg %d err %d\n",
__func__, rpc->in.ep, rpc->in.op,
cmd.fwctl_rpc.req_pa, cmd.fwctl_rpc.req_sz, cmd.fwctl_rpc.req_sg_elems,
cmd.fwctl_rpc.resp_pa, cmd.fwctl_rpc.resp_sz, cmd.fwctl_rpc.resp_sg_elems,
err);
goto done;
}
dynamic_hex_dump("out ", DUMP_PREFIX_OFFSET, 16, 1, out_payload, rpc->out.len, true);
if (copy_to_user(u64_to_user_ptr(rpc->out.payload), out_payload, rpc->out.len)) {
dev_dbg(dev, "Failed to copy out_payload to user\n");
out = ERR_PTR(-EFAULT);
goto done;
}
rpc->out.retval = le32_to_cpu(comp.fwctl_rpc.err);
*out_len = in_len;
out = in;
done:
if (out_payload_dma_addr)
dma_unmap_single(dev->parent, out_payload_dma_addr,
rpc->out.len, DMA_FROM_DEVICE);
err_out_payload:
kfree(out_payload);
if (in_payload_dma_addr)
dma_unmap_single(dev->parent, in_payload_dma_addr,
rpc->in.len, DMA_TO_DEVICE);
err_in_payload:
kfree(in_payload);
if (err)
return ERR_PTR(err);
return out;
}
static const struct fwctl_ops pdsfc_ops = {
.device_type = FWCTL_DEVICE_TYPE_PDS,
.uctx_size = sizeof(struct pdsfc_uctx),
.open_uctx = pdsfc_open_uctx,
.close_uctx = pdsfc_close_uctx,
.info = pdsfc_info,
.fw_rpc = pdsfc_fw_rpc,
};
static int pdsfc_probe(struct auxiliary_device *adev,
const struct auxiliary_device_id *id)
{
struct pds_auxiliary_dev *padev =
container_of(adev, struct pds_auxiliary_dev, aux_dev);
struct device *dev = &adev->dev;
struct pdsfc_dev *pdsfc;
int err;
pdsfc = fwctl_alloc_device(&padev->vf_pdev->dev, &pdsfc_ops,
struct pdsfc_dev, fwctl);
if (!pdsfc)
return -ENOMEM;
pdsfc->padev = padev;
err = pdsfc_identify(pdsfc);
if (err) {
fwctl_put(&pdsfc->fwctl);
return dev_err_probe(dev, err, "Failed to identify device\n");
}
err = pdsfc_init_endpoints(pdsfc);
if (err) {
fwctl_put(&pdsfc->fwctl);
return dev_err_probe(dev, err, "Failed to init endpoints\n");
}
pdsfc->caps = PDS_FWCTL_QUERY_CAP | PDS_FWCTL_SEND_CAP;
err = fwctl_register(&pdsfc->fwctl);
if (err) {
pdsfc_free_endpoints(pdsfc);
fwctl_put(&pdsfc->fwctl);
return dev_err_probe(dev, err, "Failed to register device\n");
}
auxiliary_set_drvdata(adev, pdsfc);
return 0;
}
static void pdsfc_remove(struct auxiliary_device *adev)
{
struct pdsfc_dev *pdsfc = auxiliary_get_drvdata(adev);
fwctl_unregister(&pdsfc->fwctl);
pdsfc_free_operations(pdsfc);
pdsfc_free_endpoints(pdsfc);
fwctl_put(&pdsfc->fwctl);
}
static const struct auxiliary_device_id pdsfc_id_table[] = {
{.name = PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_FWCTL_STR },
{}
};
MODULE_DEVICE_TABLE(auxiliary, pdsfc_id_table);
static struct auxiliary_driver pdsfc_driver = {
.name = "pds_fwctl",
.probe = pdsfc_probe,
.remove = pdsfc_remove,
.id_table = pdsfc_id_table,
};
module_auxiliary_driver(pdsfc_driver);
MODULE_IMPORT_NS("FWCTL");
MODULE_DESCRIPTION("pds fwctl driver");
MODULE_AUTHOR("Shannon Nelson <shannon.nelson@amd.com>");
MODULE_AUTHOR("Brett Creeley <brett.creeley@amd.com>");
MODULE_LICENSE("GPL");
