io_cq
static struct bfq_io_cq *icq_to_bic(struct io_cq *icq)
static void bfq_exit_icq(struct io_cq *icq)
struct io_cq icq; /* must be the first member */
struct io_cq *icq = hlist_entry(ioc->icq_list.first,
struct io_cq, ioc_node);
struct io_cq *icq =
list_first_entry(&q->icq_list, struct io_cq, q_node);
struct io_cq *ioc_lookup_icq(struct request_queue *q)
struct io_cq *icq;
static struct io_cq *ioc_create_icq(struct request_queue *q)
struct io_cq *icq;
static void ioc_exit_icq(struct io_cq *icq)
struct io_cq *ioc_find_get_icq(struct request_queue *q)
struct io_cq *icq = NULL;
struct io_cq *icq;
static void ioc_destroy_icq(struct io_cq *icq)
struct io_cq *ioc_find_get_icq(struct request_queue *q);
struct io_cq *ioc_lookup_icq(struct request_queue *q);
if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
WARN_ON(e->icq_align < __alignof__(struct io_cq)))
void (*init_icq)(struct io_cq *);
void (*exit_icq)(struct io_cq *);
struct io_cq;
struct ena_com_io_cq *io_cq)
create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
create_cmd.msix_vector = io_cq->msix_vector;
create_cmd.cq_depth = io_cq->q_depth;
io_cq->cdesc_addr.phys_addr);
io_cq->idx = cmd_completion.cq_idx;
io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
io_cq->numa_node_cfg_reg =
netdev_dbg(ena_dev->net_device, "Created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
struct ena_com_io_cq **io_cq)
*io_cq = &ena_dev->io_cq_queues[qid];
struct ena_com_io_cq *io_cq)
destroy_cmd.cq_idx = io_cq->idx;
struct ena_com_io_cq *io_cq;
io_cq = &ena_dev->io_cq_queues[ctx->qid];
memset(io_cq, 0x0, sizeof(*io_cq));
io_cq->q_depth = ctx->queue_size;
io_cq->direction = ctx->direction;
io_cq->qid = ctx->qid;
io_cq->msix_vector = ctx->msix_vector;
ret = ena_com_init_io_cq(ena_dev, ctx, io_cq);
ret = ena_com_create_io_cq(ena_dev, io_cq);
ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
ena_com_destroy_io_cq(ena_dev, io_cq);
ena_com_io_queue_free(ena_dev, io_sq, io_cq);
struct ena_com_io_cq *io_cq;
io_cq = &ena_dev->io_cq_queues[qid];
ena_com_destroy_io_cq(ena_dev, io_cq);
ena_com_io_queue_free(ena_dev, io_sq, io_cq);
struct ena_com_io_cq *io_cq)
memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr));
io_cq->cdesc_entry_size_in_bytes =
(io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
io_cq->cdesc_addr.virt_addr =
dma_alloc_coherent(ena_dev->dmadev, size, &io_cq->cdesc_addr.phys_addr, GFP_KERNEL);
if (!io_cq->cdesc_addr.virt_addr) {
io_cq->cdesc_addr.virt_addr =
dma_alloc_coherent(ena_dev->dmadev, size, &io_cq->cdesc_addr.phys_addr,
if (!io_cq->cdesc_addr.virt_addr) {
io_cq->phase = 1;
io_cq->head = 0;
struct ena_com_io_cq *io_cq)
if (io_cq->cdesc_addr.virt_addr) {
size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
dma_free_coherent(ena_dev->dmadev, size, io_cq->cdesc_addr.virt_addr,
io_cq->cdesc_addr.phys_addr);
io_cq->cdesc_addr.virt_addr = NULL;
static inline struct ena_com_dev *ena_com_io_cq_to_ena_dev(struct ena_com_io_cq *io_cq)
return container_of(io_cq, struct ena_com_dev, io_cq_queues[io_cq->qid]);
struct ena_com_io_cq **io_cq);
struct ena_com_io_cq *io_cq);
struct ena_com_io_cq *io_cq);
head_masked = io_cq->head & (io_cq->q_depth - 1);
expected_phase = io_cq->phase;
cdesc = (struct ena_eth_io_rx_cdesc_base *)(io_cq->cdesc_addr.virt_addr
+ (head_masked * io_cq->cdesc_entry_size_in_bytes));
ena_com_rx_cdesc_idx_to_ptr(struct ena_com_io_cq *io_cq, u16 idx)
idx &= (io_cq->q_depth - 1);
((uintptr_t)io_cq->cdesc_addr.virt_addr +
idx * io_cq->cdesc_entry_size_in_bytes);
static int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
u16 count = io_cq->cur_rx_pkt_cdesc_count, head_masked;
cdesc = ena_com_get_next_rx_cdesc(io_cq);
ena_com_cq_inc_head(io_cq);
struct ena_com_dev *dev = ena_com_io_cq_to_ena_dev(io_cq);
count, io_cq->qid, cdesc->req_id);
*first_cdesc_idx = io_cq->cur_rx_pkt_cdesc_start_idx;
head_masked = io_cq->head & (io_cq->q_depth - 1);
io_cq->cur_rx_pkt_cdesc_count = 0;
io_cq->cur_rx_pkt_cdesc_start_idx = head_masked;
netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
io_cq->qid, *first_cdesc_idx, count);
io_cq->cur_rx_pkt_cdesc_count = count;
static void ena_com_rx_set_flags(struct ena_com_io_cq *io_cq,
netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
u16 q_depth = io_cq->q_depth;
WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, "wrong Q type");
rc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx, &nb_hw_desc);
netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
"Fetch rx packet: queue %d completed desc: %d\n", io_cq->qid, nb_hw_desc);
netdev_err(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx);
cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i);
netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
ena_com_rx_set_flags(io_cq, ena_rx_ctx, cdesc);
bool ena_com_cq_empty(struct ena_com_io_cq *io_cq)
cdesc = ena_com_get_next_rx_cdesc(io_cq);
struct ena_com_io_cq *io_cq)
static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq,
if (!io_cq->numa_node_cfg_reg)
writel(numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)
io_cq->head++;
if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))
io_cq->phase ^= 1;
static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq,
masked_head = io_cq->head & (io_cq->q_depth - 1);
expected_phase = io_cq->phase;
((uintptr_t)io_cq->cdesc_addr.virt_addr +
(masked_head * io_cq->cdesc_entry_size_in_bytes));
if (unlikely(*req_id >= io_cq->q_depth)) {
netdev_err(ena_com_io_cq_to_ena_dev(io_cq)->net_device, "Invalid req id %d\n",
ena_com_cq_inc_head(io_cq);
int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);
static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
writel(intr_reg->intr_control, io_cq->unmask_reg);
qp = phba->sli4_hba.hdwq[eqidx].io_cq;
qp = phba->sli4_hba.hdwq[qidx].io_cq;
cq = phba->sli4_hba.hdwq[wqidx].io_cq;
if (phba->sli4_hba.hdwq[cq_idx].io_cq->queue_id == qid)
lpfc_debug_dump_q(phba->sli4_hba.hdwq[cq_idx].io_cq);
phba->sli4_hba.hdwq[idx].io_cq = qdesc;
lpfc_sli4_queue_free(hdwq[idx].io_cq);
hdwq[idx].io_cq = NULL;
qp[qidx].io_cq,
lpfc_cq_destroy(phba, qp->io_cq);
sli4_hba->sli4_write_cq_db(phba, qp->io_cq, 0,
struct lpfc_queue *io_cq; /* Fast-path FCP & NVME compl queue */
struct io_cq *icq;
struct io_cq __rcu *icq_hint;