drivers/net/ethernet/amazon/ena/ena_netdev.c

223

rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];

drivers/net/ethernet/amazon/ena/ena_netdev.h

247

struct ena_ring *xdp_ring;

drivers/net/ethernet/amazon/ena/ena_xdp.h

105

xdp_ring = rx_ring->xdp_ring;

drivers/net/ethernet/amazon/ena/ena_xdp.h

108

spin_lock(&xdp_ring->xdp_tx_lock);

drivers/net/ethernet/amazon/ena/ena_xdp.h

110

if (ena_xdp_xmit_frame(xdp_ring, rx_ring->adapter, xdpf,

drivers/net/ethernet/amazon/ena/ena_xdp.h

114

spin_unlock(&xdp_ring->xdp_tx_lock);

drivers/net/ethernet/amazon/ena/ena_xdp.h

86

struct ena_ring *xdp_ring;

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

272

struct i40e_ring *xdp_ring = READ_ONCE(vsi->xdp_rings[i]);

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

274

if (!xdp_ring)

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

279

i, *xdp_ring->state,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

280

xdp_ring->queue_index,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

281

xdp_ring->reg_idx);

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

285

xdp_ring->next_to_use,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

286

xdp_ring->next_to_clean,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

287

xdp_ring->ring_active);

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

290

i, xdp_ring->stats.packets,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

291

xdp_ring->stats.bytes,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

292

xdp_ring->tx_stats.restart_queue);

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

296

xdp_ring->tx_stats.tx_busy,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

297

xdp_ring->tx_stats.tx_done_old);

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

300

i, xdp_ring->size);

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

303

i, xdp_ring->dcb_tc);

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

306

i, xdp_ring->itr_setting,

drivers/net/ethernet/intel/i40e/i40e_debugfs.c

307

ITR_IS_DYNAMIC(xdp_ring->itr_setting) ?

drivers/net/ethernet/intel/i40e/i40e_main.c

4543

struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];

drivers/net/ethernet/intel/i40e/i40e_main.c

4545

xdp_ring->q_vector = q_vector;

drivers/net/ethernet/intel/i40e/i40e_main.c

4546

xdp_ring->next = q_vector->tx.ring;

drivers/net/ethernet/intel/i40e/i40e_main.c

4547

q_vector->tx.ring = xdp_ring;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2245

struct i40e_ring *xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2247

int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring)

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2254

return i40e_xmit_xdp_ring(xdpf, xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2266

struct i40e_ring *xdp_ring;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2279

xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2280

result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2311

void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring)

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2317

writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2355

struct i40e_ring *xdp_ring =

drivers/net/ethernet/intel/i40e/i40e_txrx.c

2358

i40e_xdp_ring_update_tail(xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3777

struct i40e_ring *xdp_ring)

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3781

u16 i = 0, index = xdp_ring->next_to_use;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3782

struct i40e_tx_buffer *tx_head = &xdp_ring->tx_bi[index];

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3784

struct i40e_tx_desc *tx_desc = I40E_TX_DESC(xdp_ring, index);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3788

if (unlikely(I40E_DESC_UNUSED(xdp_ring) < 1 + nr_frags)) {

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3789

xdp_ring->tx_stats.tx_busy++;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3800

dma = dma_map_single(xdp_ring->dev, data, size, DMA_TO_DEVICE);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3801

if (dma_mapping_error(xdp_ring->dev, dma))

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3812

if (++index == xdp_ring->count)

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3818

tx_bi = &xdp_ring->tx_bi[index];

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3819

tx_desc = I40E_TX_DESC(xdp_ring, index);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3834

xdp_ring->xdp_tx_active++;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3837

xdp_ring->next_to_use = index;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3843

tx_bi = &xdp_ring->tx_bi[index];

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3845

dma_unmap_page(xdp_ring->dev,

drivers/net/ethernet/intel/i40e/i40e_txrx.c

3854

index += xdp_ring->count;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

4011

struct i40e_ring *xdp_ring;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

4025

xdp_ring = vsi->xdp_rings[queue_index];

drivers/net/ethernet/intel/i40e/i40e_txrx.c

4031

err = i40e_xmit_xdp_ring(xdpf, xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_txrx.c

4038

i40e_xdp_ring_update_tail(xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_txrx_common.h

14

void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_txrx_common.h

9

int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

202

struct i40e_ring *xdp_ring;

drivers/net/ethernet/intel/i40e/i40e_xsk.c

222

xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];

drivers/net/ethernet/intel/i40e/i40e_xsk.c

223

result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

508

static void i40e_xmit_pkt(struct i40e_ring *xdp_ring, struct xdp_desc *desc,

drivers/net/ethernet/intel/i40e/i40e_xsk.c

515

dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc->addr);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

516

xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc->len);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

518

tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use++);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

525

static void i40e_xmit_pkt_batch(struct i40e_ring *xdp_ring, struct xdp_desc *desc,

drivers/net/ethernet/intel/i40e/i40e_xsk.c

528

u16 ntu = xdp_ring->next_to_use;

drivers/net/ethernet/intel/i40e/i40e_xsk.c

537

dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc[i].addr);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

538

xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc[i].len);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

540

tx_desc = I40E_TX_DESC(xdp_ring, ntu++);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

547

xdp_ring->next_to_use = ntu;

drivers/net/ethernet/intel/i40e/i40e_xsk.c

550

static void i40e_fill_tx_hw_ring(struct i40e_ring *xdp_ring, struct xdp_desc *descs, u32 nb_pkts,

drivers/net/ethernet/intel/i40e/i40e_xsk.c

558

i40e_xmit_pkt_batch(xdp_ring, &descs[i], total_bytes);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

560

i40e_xmit_pkt(xdp_ring, &descs[i], total_bytes);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

563

static void i40e_set_rs_bit(struct i40e_ring *xdp_ring)

drivers/net/ethernet/intel/i40e/i40e_xsk.c

565

u16 ntu = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : xdp_ring->count - 1;

drivers/net/ethernet/intel/i40e/i40e_xsk.c

568

tx_desc = I40E_TX_DESC(xdp_ring, ntu);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

579

static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)

drivers/net/ethernet/intel/i40e/i40e_xsk.c

581

struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs;

drivers/net/ethernet/intel/i40e/i40e_xsk.c

585

nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

589

if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) {

drivers/net/ethernet/intel/i40e/i40e_xsk.c

590

nb_processed = xdp_ring->count - xdp_ring->next_to_use;

drivers/net/ethernet/intel/i40e/i40e_xsk.c

591

i40e_fill_tx_hw_ring(xdp_ring, descs, nb_processed, &total_bytes);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

592

xdp_ring->next_to_use = 0;

drivers/net/ethernet/intel/i40e/i40e_xsk.c

595

i40e_fill_tx_hw_ring(xdp_ring, &descs[nb_processed], nb_pkts - nb_processed,

drivers/net/ethernet/intel/i40e/i40e_xsk.c

599

i40e_set_rs_bit(xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

600

i40e_xdp_ring_update_tail(xdp_ring);

drivers/net/ethernet/intel/i40e/i40e_xsk.c

602

i40e_update_tx_stats(xdp_ring, nb_pkts, total_bytes);

drivers/net/ethernet/intel/ice/ice.h

835

ring = vsi->rx_rings[qid]->xdp_ring;

drivers/net/ethernet/intel/ice/ice_base.c

1464

struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx];

drivers/net/ethernet/intel/ice/ice_base.c

1467

ice_fill_txq_meta(vsi, xdp_ring, &txq_meta);

drivers/net/ethernet/intel/ice/ice_base.c

1468

err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring,

drivers/net/ethernet/intel/ice/ice_base.c

1500

struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx];

drivers/net/ethernet/intel/ice/ice_base.c

1505

ice_set_ring_xdp(xdp_ring);

drivers/net/ethernet/intel/ice/ice_main.c

2610

struct ice_tx_ring *xdp_ring;

drivers/net/ethernet/intel/ice/ice_main.c

2612

xdp_ring = kzalloc_obj(*xdp_ring);

drivers/net/ethernet/intel/ice/ice_main.c

2613

if (!xdp_ring)

drivers/net/ethernet/intel/ice/ice_main.c

2618

ice_free_tx_ring(xdp_ring);

drivers/net/ethernet/intel/ice/ice_main.c

2622

xdp_ring->ring_stats = ring_stats;

drivers/net/ethernet/intel/ice/ice_main.c

2623

xdp_ring->q_index = xdp_q_idx;

drivers/net/ethernet/intel/ice/ice_main.c

2624

xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx];

drivers/net/ethernet/intel/ice/ice_main.c

2625

xdp_ring->vsi = vsi;

drivers/net/ethernet/intel/ice/ice_main.c

2626

xdp_ring->netdev = NULL;

drivers/net/ethernet/intel/ice/ice_main.c

2627

xdp_ring->dev = dev;

drivers/net/ethernet/intel/ice/ice_main.c

2628

xdp_ring->count = vsi->num_tx_desc;

drivers/net/ethernet/intel/ice/ice_main.c

2629

WRITE_ONCE(vsi->xdp_rings[i], xdp_ring);

drivers/net/ethernet/intel/ice/ice_main.c

2630

if (ice_setup_tx_ring(xdp_ring))

drivers/net/ethernet/intel/ice/ice_main.c

2632

ice_set_ring_xdp(xdp_ring);

drivers/net/ethernet/intel/ice/ice_main.c

2633

spin_lock_init(&xdp_ring->tx_lock);

drivers/net/ethernet/intel/ice/ice_main.c

2634

for (j = 0; j < xdp_ring->count; j++) {

drivers/net/ethernet/intel/ice/ice_main.c

2635

tx_desc = ICE_TX_DESC(xdp_ring, j);

drivers/net/ethernet/intel/ice/ice_main.c

2709

struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_id];

drivers/net/ethernet/intel/ice/ice_main.c

2711

xdp_ring->q_vector = q_vector;

drivers/net/ethernet/intel/ice/ice_main.c

2712

xdp_ring->next = q_vector->tx.tx_ring;

drivers/net/ethernet/intel/ice/ice_main.c

2713

q_vector->tx.tx_ring = xdp_ring;

drivers/net/ethernet/intel/ice/ice_main.c

2719

vsi->rx_rings[q_idx]->xdp_ring = ice_xdp_ring_from_qid(vsi,

drivers/net/ethernet/intel/ice/ice_txrx.c

1030

xdp_verdict = ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_desc);

drivers/net/ethernet/intel/ice/ice_txrx.c

1074

ice_finalize_xdp_rx(xdp_ring, xdp_xmit, cached_ntu);

drivers/net/ethernet/intel/ice/ice_txrx.c

652

struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_txrx.c

669

spin_lock(&xdp_ring->tx_lock);

drivers/net/ethernet/intel/ice/ice_txrx.c

670

ret = __ice_xmit_xdp_ring(&xdp->base, xdp_ring, false);

drivers/net/ethernet/intel/ice/ice_txrx.c

672

spin_unlock(&xdp_ring->tx_lock);

drivers/net/ethernet/intel/ice/ice_txrx.c

703

struct ice_tx_ring *xdp_ring)

drivers/net/ethernet/intel/ice/ice_txrx.c

713

return __ice_xmit_xdp_ring(&xdp, xdp_ring, true);

drivers/net/ethernet/intel/ice/ice_txrx.c

735

struct ice_tx_ring *xdp_ring;

drivers/net/ethernet/intel/ice/ice_txrx.c

750

xdp_ring = vsi->xdp_rings[queue_index];

drivers/net/ethernet/intel/ice/ice_txrx.c

751

spin_lock(&xdp_ring->tx_lock);

drivers/net/ethernet/intel/ice/ice_txrx.c

756

xdp_ring = vsi->xdp_rings[queue_index];

drivers/net/ethernet/intel/ice/ice_txrx.c

759

tx_buf = &xdp_ring->tx_buf[xdp_ring->next_to_use];

drivers/net/ethernet/intel/ice/ice_txrx.c

764

err = ice_xmit_xdp_ring(xdpf, xdp_ring);

drivers/net/ethernet/intel/ice/ice_txrx.c

770

tx_buf->rs_idx = ice_set_rs_bit(xdp_ring);

drivers/net/ethernet/intel/ice/ice_txrx.c

772

ice_xdp_ring_update_tail(xdp_ring);

drivers/net/ethernet/intel/ice/ice_txrx.c

775

spin_unlock(&xdp_ring->tx_lock);

drivers/net/ethernet/intel/ice/ice_txrx.c

948

struct ice_tx_ring *xdp_ring = NULL;

drivers/net/ethernet/intel/ice/ice_txrx.c

961

xdp_ring = rx_ring->xdp_ring;

drivers/net/ethernet/intel/ice/ice_txrx.c

962

cached_ntu = xdp_ring->next_to_use;

drivers/net/ethernet/intel/ice/ice_txrx.h

296

struct ice_tx_ring *xdp_ring;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

293

static u32 ice_clean_xdp_irq(struct ice_tx_ring *xdp_ring)

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

296

struct device *dev = xdp_ring->dev;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

297

u32 ntc = xdp_ring->next_to_clean;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

299

u32 cnt = xdp_ring->count;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

306

idx = xdp_ring->tx_buf[ntc].rs_idx;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

307

tx_desc = ICE_TX_DESC(xdp_ring, idx);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

324

struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

340

tx_buf = &xdp_ring->tx_buf[ntc];

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

355

xdp_ring->next_to_clean = ntc;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

356

xdp_ring->xdp_tx_active -= xdp_tx;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

357

ice_update_tx_ring_stats(xdp_ring, total_pkts, total_bytes);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

368

int __ice_xmit_xdp_ring(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

373

struct device *dev = xdp_ring->dev;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

374

u32 ntu = xdp_ring->next_to_use;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

378

u32 cnt = xdp_ring->count;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

386

free_space = ICE_DESC_UNUSED(xdp_ring);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

387

if (free_space < ICE_RING_QUARTER(xdp_ring))

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

388

free_space += ice_clean_xdp_irq(xdp_ring);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

400

tx_desc = ICE_TX_DESC(xdp_ring, ntu);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

401

tx_head = &xdp_ring->tx_buf[ntu];

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

436

tx_desc = ICE_TX_DESC(xdp_ring, ntu);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

437

tx_buf = &xdp_ring->tx_buf[ntu];

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

459

xdp_ring->xdp_tx_active++;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

460

xdp_ring->next_to_use = ntu;

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

466

tx_buf = &xdp_ring->tx_buf[ntu];

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

480

ice_stats_inc(xdp_ring->ring_stats, tx_busy);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

495

void ice_finalize_xdp_rx(struct ice_tx_ring *xdp_ring, unsigned int xdp_res,

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

498

struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[first_idx];

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

505

spin_lock(&xdp_ring->tx_lock);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

509

tx_buf->rs_idx = ice_set_rs_bit(xdp_ring);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

510

ice_xdp_ring_update_tail(xdp_ring);

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

512

spin_unlock(&xdp_ring->tx_lock);

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

100

static inline void ice_xdp_ring_update_tail(struct ice_tx_ring *xdp_ring)

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

106

writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail);

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

115

static inline u32 ice_set_rs_bit(const struct ice_tx_ring *xdp_ring)

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

117

u32 rs_idx = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : xdp_ring->count - 1;

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

120

tx_desc = ICE_TX_DESC(xdp_ring, rs_idx);

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

127

void ice_finalize_xdp_rx(struct ice_tx_ring *xdp_ring, unsigned int xdp_res, u32 first_idx);

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

128

int __ice_xmit_xdp_ring(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

231

napi_schedule(&vsi->rx_rings[qid]->xdp_ring->q_vector->napi);

drivers/net/ethernet/intel/ice/ice_xsk.c

362

static u32 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

365

u16 ntc = xdp_ring->next_to_clean;

drivers/net/ethernet/intel/ice/ice_xsk.c

367

u16 cnt = xdp_ring->count;

drivers/net/ethernet/intel/ice/ice_xsk.c

374

last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;

drivers/net/ethernet/intel/ice/ice_xsk.c

375

tx_desc = ICE_TX_DESC(xdp_ring, last_rs);

drivers/net/ethernet/intel/ice/ice_xsk.c

387

if (likely(!xdp_ring->xdp_tx_active)) {

drivers/net/ethernet/intel/ice/ice_xsk.c

392

ntc = xdp_ring->next_to_clean;

drivers/net/ethernet/intel/ice/ice_xsk.c

394

tx_buf = &xdp_ring->tx_buf[ntc];

drivers/net/ethernet/intel/ice/ice_xsk.c

399

xdp_ring->xdp_tx_active--;

drivers/net/ethernet/intel/ice/ice_xsk.c

405

if (ntc >= xdp_ring->count)

drivers/net/ethernet/intel/ice/ice_xsk.c

410

xdp_ring->next_to_clean += completed_frames;

drivers/net/ethernet/intel/ice/ice_xsk.c

411

if (xdp_ring->next_to_clean >= cnt)

drivers/net/ethernet/intel/ice/ice_xsk.c

412

xdp_ring->next_to_clean -= cnt;

drivers/net/ethernet/intel/ice/ice_xsk.c

433

struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

438

u32 ntu = xdp_ring->next_to_use;

drivers/net/ethernet/intel/ice/ice_xsk.c

446

free_space = ICE_DESC_UNUSED(xdp_ring);

drivers/net/ethernet/intel/ice/ice_xsk.c

447

if (free_space < ICE_RING_QUARTER(xdp_ring))

drivers/net/ethernet/intel/ice/ice_xsk.c

448

free_space += ice_clean_xdp_irq_zc(xdp_ring, xsk_pool);

drivers/net/ethernet/intel/ice/ice_xsk.c

460

tx_desc = ICE_TX_DESC(xdp_ring, ntu);

drivers/net/ethernet/intel/ice/ice_xsk.c

461

tx_buf = &xdp_ring->tx_buf[ntu];

drivers/net/ethernet/intel/ice/ice_xsk.c

475

xdp_ring->xdp_tx_active++;

drivers/net/ethernet/intel/ice/ice_xsk.c

477

if (++ntu == xdp_ring->count)

drivers/net/ethernet/intel/ice/ice_xsk.c

483

tx_desc = ICE_TX_DESC(xdp_ring, ntu);

drivers/net/ethernet/intel/ice/ice_xsk.c

484

tx_buf = &xdp_ring->tx_buf[ntu];

drivers/net/ethernet/intel/ice/ice_xsk.c

491

xdp_ring->next_to_use = ntu;

drivers/net/ethernet/intel/ice/ice_xsk.c

499

ice_stats_inc(xdp_ring->ring_stats, tx_busy);

drivers/net/ethernet/intel/ice/ice_xsk.c

516

struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

539

result = ice_xmit_xdp_tx_zc(xdp, xdp_ring, xsk_pool);

drivers/net/ethernet/intel/ice/ice_xsk.c

575

struct ice_tx_ring *xdp_ring;

drivers/net/ethernet/intel/ice/ice_xsk.c

586

xdp_ring = rx_ring->xdp_ring;

drivers/net/ethernet/intel/ice/ice_xsk.c

634

xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

684

ice_finalize_xdp_rx(xdp_ring, xdp_xmit, 0);

drivers/net/ethernet/intel/ice/ice_xsk.c

709

static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

719

tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use++);

drivers/net/ethernet/intel/ice/ice_xsk.c

734

static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

739

u16 ntu = xdp_ring->next_to_use;

drivers/net/ethernet/intel/ice/ice_xsk.c

750

tx_desc = ICE_TX_DESC(xdp_ring, ntu++);

drivers/net/ethernet/intel/ice/ice_xsk.c

758

xdp_ring->next_to_use = ntu;

drivers/net/ethernet/intel/ice/ice_xsk.c

769

static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.c

779

ice_xmit_pkt_batch(xdp_ring, xsk_pool, &descs[i], total_bytes);

drivers/net/ethernet/intel/ice/ice_xsk.c

781

ice_xmit_pkt(xdp_ring, xsk_pool, &descs[i], total_bytes);

drivers/net/ethernet/intel/ice/ice_xsk.c

791

bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, struct xsk_buff_pool *xsk_pool)

drivers/net/ethernet/intel/ice/ice_xsk.c

798

ice_clean_xdp_irq_zc(xdp_ring, xsk_pool);

drivers/net/ethernet/intel/ice/ice_xsk.c

800

if (!netif_carrier_ok(xdp_ring->vsi->netdev) ||

drivers/net/ethernet/intel/ice/ice_xsk.c

801

!netif_running(xdp_ring->vsi->netdev))

drivers/net/ethernet/intel/ice/ice_xsk.c

804

budget = ICE_DESC_UNUSED(xdp_ring);

drivers/net/ethernet/intel/ice/ice_xsk.c

805

budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring));

drivers/net/ethernet/intel/ice/ice_xsk.c

811

if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) {

drivers/net/ethernet/intel/ice/ice_xsk.c

812

nb_processed = xdp_ring->count - xdp_ring->next_to_use;

drivers/net/ethernet/intel/ice/ice_xsk.c

813

ice_fill_tx_hw_ring(xdp_ring, xsk_pool, descs, nb_processed,

drivers/net/ethernet/intel/ice/ice_xsk.c

815

xdp_ring->next_to_use = 0;

drivers/net/ethernet/intel/ice/ice_xsk.c

818

ice_fill_tx_hw_ring(xdp_ring, xsk_pool, &descs[nb_processed],

drivers/net/ethernet/intel/ice/ice_xsk.c

821

ice_set_rs_bit(xdp_ring);

drivers/net/ethernet/intel/ice/ice_xsk.c

822

ice_xdp_ring_update_tail(xdp_ring);

drivers/net/ethernet/intel/ice/ice_xsk.c

823

ice_update_tx_ring_stats(xdp_ring, nb_pkts, total_bytes);

drivers/net/ethernet/intel/ice/ice_xsk.c

857

ring = vsi->rx_rings[queue_id]->xdp_ring;

drivers/net/ethernet/intel/ice/ice_xsk.c

919

void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring)

drivers/net/ethernet/intel/ice/ice_xsk.c

921

u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use;

drivers/net/ethernet/intel/ice/ice_xsk.c

925

struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];

drivers/net/ethernet/intel/ice/ice_xsk.c

935

if (ntc >= xdp_ring->count)

drivers/net/ethernet/intel/ice/ice_xsk.c

940

xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);

drivers/net/ethernet/intel/ice/ice_xsk.h

23

void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring);

drivers/net/ethernet/intel/ice/ice_xsk.h

24

bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, struct xsk_buff_pool *xsk_pool);

drivers/net/ethernet/intel/ice/ice_xsk.h

34

static inline bool ice_xmit_zc(struct ice_tx_ring __always_unused *xdp_ring,

drivers/net/ethernet/intel/ice/ice_xsk.h

77

static inline void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring) { }

drivers/net/ethernet/intel/ixgbe/ixgbe.h

702

struct ixgbe_ring *xdp_ring[IXGBE_MAX_XDP_QS];

drivers/net/ethernet/intel/ixgbe/ixgbe.h

873

return adapter->xdp_ring[index];

drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c

1272

adapter->xdp_ring[i]->count = new_tx_count;

drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c

1316

memcpy(&temp_ring[i], adapter->xdp_ring[j],

drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c

1337

ixgbe_free_tx_resources(adapter->xdp_ring[j]);

drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c

1339

memcpy(adapter->xdp_ring[j], &temp_ring[i],

drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c

1025

WRITE_ONCE(adapter->xdp_ring[ring->queue_index], NULL);

drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c

1111

if (adapter->xdp_ring[i])

drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c

1112

adapter->xdp_ring[i]->ring_idx = i;

drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c

265

adapter->xdp_ring[i]->reg_idx = reg_idx;

drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c

953

WRITE_ONCE(adapter->xdp_ring[xdp_idx], ring);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

1018

struct ixgbe_ring *xdp_ring = adapter->xdp_ring[i];

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

1020

tc = xdp_ring->dcb_tc;

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

1022

clear_bit(__IXGBE_HANG_CHECK_ARMED, &xdp_ring->state);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

10958

if (adapter->xdp_ring[i]->xsk_pool)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11235

struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11239

xdp_ring = adapter->xdp_ring[ring];

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11247

if (xdp_ring)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11248

ixgbe_disable_txr(adapter, xdp_ring);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11251

if (xdp_ring)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11255

if (xdp_ring)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11256

ixgbe_clean_tx_ring(xdp_ring);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11260

if (xdp_ring)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11261

ixgbe_reset_txr_stats(xdp_ring);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11275

struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11279

xdp_ring = adapter->xdp_ring[ring];

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11282

if (xdp_ring)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11283

ixgbe_configure_tx_ring(adapter, xdp_ring);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11287

if (xdp_ring)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11288

clear_bit(__IXGBE_TX_DISABLED, &xdp_ring->state);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

11882

u64_stats_init(&adapter->xdp_ring[i]->syncp);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

4073

ixgbe_configure_tx_ring(adapter, adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

634

ring = adapter->xdp_ring[n];

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

6442

struct ixgbe_ring *ring = adapter->xdp_ring[i];

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

6489

struct ixgbe_ring *ring = adapter->xdp_ring[i];

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

6677

ixgbe_clean_tx_ring(adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

6718

if (adapter->xdp_ring[0])

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7135

err = ixgbe_setup_tx_resources(adapter->xdp_ring[j]);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7147

ixgbe_free_tx_resources(adapter->xdp_ring[j]);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7292

if (adapter->xdp_ring[i]->desc)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7293

ixgbe_free_tx_resources(adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7726

struct ixgbe_ring *xdp_ring = READ_ONCE(adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7728

if (!xdp_ring)

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7730

restart_queue += xdp_ring->tx_stats.restart_queue;

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7731

tx_busy += xdp_ring->tx_stats.tx_busy;

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7732

bytes += xdp_ring->stats.bytes;

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7733

packets += xdp_ring->stats.packets;

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

7960

&adapter->xdp_ring[i]->state);

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

9729

struct ixgbe_ring *ring = READ_ONCE(adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

391

static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget)

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

393

struct xsk_buff_pool *pool = xdp_ring->xsk_pool;

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

402

if (unlikely(!ixgbe_desc_unused(xdp_ring))) {

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

407

if (!netif_carrier_ok(xdp_ring->netdev))

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

416

tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

421

tx_desc = IXGBE_TX_DESC(xdp_ring, xdp_ring->next_to_use);

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

433

xdp_ring->next_to_use++;

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

434

if (xdp_ring->next_to_use == xdp_ring->count)

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

435

xdp_ring->next_to_use = 0;

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

441

ixgbe_xdp_ring_update_tail(xdp_ring);

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

525

ring = adapter->xdp_ring[qid];

drivers/net/ethernet/intel/ixgbevf/ethtool.c

269

adapter->xdp_ring[i]->count = new_tx_count;

drivers/net/ethernet/intel/ixgbevf/ethtool.c

307

tx_ring[i] = *adapter->xdp_ring[j];

drivers/net/ethernet/intel/ixgbevf/ethtool.c

368

ixgbevf_free_tx_resources(adapter->xdp_ring[j]);

drivers/net/ethernet/intel/ixgbevf/ethtool.c

369

*adapter->xdp_ring[j] = tx_ring[i];

drivers/net/ethernet/intel/ixgbevf/ethtool.c

470

ring = adapter->xdp_ring[j];

drivers/net/ethernet/intel/ixgbevf/ixgbevf.h

320

struct ixgbevf_ring *xdp_ring[MAX_XDP_QUEUES];

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

1053

struct ixgbevf_ring *xdp_ring;

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

1067

xdp_ring = adapter->xdp_ring[rx_ring->queue_index];

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

1068

result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

1243

struct ixgbevf_ring *xdp_ring =

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

1244

adapter->xdp_ring[rx_ring->queue_index];

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

1250

ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

1763

ixgbevf_configure_tx_ring(adapter, adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

2516

ixgbevf_clean_tx_ring(adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

2556

u8 reg_idx = adapter->xdp_ring[i]->reg_idx;

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

2829

adapter->xdp_ring[xdp_idx] = ring;

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

2886

adapter->xdp_ring[ring->queue_index] = NULL;

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

3262

set_check_for_tx_hang(adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

3435

if (adapter->xdp_ring[i]->desc)

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

3436

ixgbevf_free_tx_resources(adapter->xdp_ring[i]);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

3498

err = ixgbevf_setup_tx_resources(adapter->xdp_ring[j]);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

3509

ixgbevf_free_tx_resources(adapter->xdp_ring[j]);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

4430

ring = adapter->xdp_ring[i];

drivers/net/ethernet/netronome/nfp/nfd3/dp.c

922

tx_ring = r_vec->xdp_ring;

drivers/net/ethernet/netronome/nfp/nfd3/xsk.c

126

tx_ring = r_vec->xdp_ring;

drivers/net/ethernet/netronome/nfp/nfd3/xsk.c

399

if (!nfp_nfd3_xsk_complete(r_vec->xdp_ring))

drivers/net/ethernet/netronome/nfp/nfd3/xsk.c

402

nfp_nfd3_xsk_tx(r_vec->xdp_ring);

drivers/net/ethernet/netronome/nfp/nfdk/dp.c

1044

tx_ring = r_vec->xdp_ring;

drivers/net/ethernet/netronome/nfp/nfp_net.h

419

struct nfp_net_tx_ring *xdp_ring;

drivers/net/ethernet/netronome/nfp/nfp_net_common.c

799

r_vec->xdp_ring = idx < dp->num_tx_rings - dp->num_stack_tx_rings ?

drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c

90

tx_ring = r_vec->xdp_ring;

drivers/net/veth.c

1041

err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);

drivers/net/veth.c

1057

ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free);

drivers/net/veth.c

1091

ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free);

drivers/net/veth.c

314

if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb)))

drivers/net/veth.c

531

spin_lock(&rq->xdp_ring.producer_lock);

drivers/net/veth.c

537

__ptr_ring_produce(&rq->xdp_ring, ptr)))

drivers/net/veth.c

541

spin_unlock(&rq->xdp_ring.producer_lock);

drivers/net/veth.c

69

struct ptr_ring xdp_ring;

drivers/net/veth.c

909

void *ptr = __ptr_ring_consume(&rq->xdp_ring);

drivers/net/veth.c

989

if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) {

net/xdp/xsk_queue.h

30

struct xdp_ring ptrs;

net/xdp/xsk_queue.h

36

struct xdp_ring ptrs;

net/xdp/xsk_queue.h

45

struct xdp_ring *ring;