arch/parisc/include/asm/pci.h

59

struct ioc *iommu; /* IOMMU this device is under */

arch/powerpc/include/asm/pci-bridge.h

138

struct iommu_device iommu;

arch/powerpc/kernel/iommu.c

1242

return &hose->iommu;

arch/powerpc/kernel/iommu.c

1288

iommu_device_sysfs_add(&phb->iommu, phb->parent,

arch/powerpc/kernel/iommu.c

1291

iommu_device_register(&phb->iommu, &spapr_tce_iommu_ops,

arch/powerpc/kernel/iommu.c

1297

iommu_device_unregister(&phb->iommu);

arch/powerpc/kernel/iommu.c

1298

iommu_device_sysfs_remove(&phb->iommu);

arch/sparc/include/asm/device.h

14

void *iommu;

arch/sparc/include/asm/iommu-common.h

35

extern void iommu_tbl_pool_init(struct iommu_map_table *iommu,

arch/sparc/include/asm/iommu-common.h

43

struct iommu_map_table *iommu,

arch/sparc/include/asm/iommu-common.h

49

extern void iommu_tbl_range_free(struct iommu_map_table *iommu,

arch/sparc/include/asm/iommu_64.h

89

int iommu_table_init(struct iommu *iommu, int tsbsize,

arch/sparc/kernel/iommu-common.c

109

unsigned int npools = iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

111

bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0);

arch/sparc/kernel/iommu-common.c

126

pool = &(iommu->large_pool);

arch/sparc/kernel/iommu-common.c

131

pool = &(iommu->pools[pool_nr]);

arch/sparc/kernel/iommu-common.c

152

shift = iommu->table_map_base >> iommu->table_shift;

arch/sparc/kernel/iommu-common.c

161

pool = &(iommu->pools[0]);

arch/sparc/kernel/iommu-common.c

174

if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) {

arch/sparc/kernel/iommu-common.c

176

boundary_size = iommu->poolsize * iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

179

iommu->table_shift);

arch/sparc/kernel/iommu-common.c

181

n = iommu_area_alloc(iommu->map, limit, start, npages, shift,

arch/sparc/kernel/iommu-common.c

187

set_flush(iommu);

arch/sparc/kernel/iommu-common.c

19

static inline bool need_flush(struct iommu_map_table *iommu)

arch/sparc/kernel/iommu-common.c

190

} else if (!largealloc && pass <= iommu->nr_pools) {

arch/sparc/kernel/iommu-common.c

192

pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);

arch/sparc/kernel/iommu-common.c

193

pool = &(iommu->pools[pool_nr]);

arch/sparc/kernel/iommu-common.c

196

set_flush(iommu);

arch/sparc/kernel/iommu-common.c

205

if (iommu->lazy_flush &&

arch/sparc/kernel/iommu-common.c

206

(n < pool->hint || need_flush(iommu))) {

arch/sparc/kernel/iommu-common.c

207

clear_flush(iommu);

arch/sparc/kernel/iommu-common.c

208

iommu->lazy_flush(iommu);

arch/sparc/kernel/iommu-common.c

21

return ((iommu->flags & IOMMU_NEED_FLUSH) != 0);

arch/sparc/kernel/iommu-common.c

24

static inline void set_flush(struct iommu_map_table *iommu)

arch/sparc/kernel/iommu-common.c

246

void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr,

arch/sparc/kernel/iommu-common.c

251

unsigned long shift = iommu->table_shift;

arch/sparc/kernel/iommu-common.c

254

entry = (dma_addr - iommu->table_map_base) >> shift;

arch/sparc/kernel/iommu-common.c

255

pool = get_pool(iommu, entry);

arch/sparc/kernel/iommu-common.c

258

bitmap_clear(iommu->map, entry, npages);

arch/sparc/kernel/iommu-common.c

26

iommu->flags |= IOMMU_NEED_FLUSH;

arch/sparc/kernel/iommu-common.c

29

static inline void clear_flush(struct iommu_map_table *iommu)

arch/sparc/kernel/iommu-common.c

31

iommu->flags &= ~IOMMU_NEED_FLUSH;

arch/sparc/kernel/iommu-common.c

52

void iommu_tbl_pool_init(struct iommu_map_table *iommu,

arch/sparc/kernel/iommu-common.c

60

struct iommu_pool *p = &(iommu->large_pool);

arch/sparc/kernel/iommu-common.c

64

iommu->nr_pools = IOMMU_NR_POOLS;

arch/sparc/kernel/iommu-common.c

66

iommu->nr_pools = npools;

arch/sparc/kernel/iommu-common.c

69

iommu->table_shift = table_shift;

arch/sparc/kernel/iommu-common.c

70

iommu->lazy_flush = lazy_flush;

arch/sparc/kernel/iommu-common.c

73

iommu->flags |= IOMMU_NO_SPAN_BOUND;

arch/sparc/kernel/iommu-common.c

75

iommu->flags |= IOMMU_HAS_LARGE_POOL;

arch/sparc/kernel/iommu-common.c

78

iommu->poolsize = num_entries/iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

80

iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

81

for (i = 0; i < iommu->nr_pools; i++) {

arch/sparc/kernel/iommu-common.c

82

spin_lock_init(&(iommu->pools[i].lock));

arch/sparc/kernel/iommu-common.c

83

iommu->pools[i].start = start;

arch/sparc/kernel/iommu-common.c

84

iommu->pools[i].hint = start;

arch/sparc/kernel/iommu-common.c

85

start += iommu->poolsize; /* start for next pool */

arch/sparc/kernel/iommu-common.c

86

iommu->pools[i].end = start - 1;

arch/sparc/kernel/iommu-common.c

98

struct iommu_map_table *iommu,

arch/sparc/kernel/iommu.c

103

spin_lock_init(&iommu->lock);

arch/sparc/kernel/iommu.c

104

iommu->ctx_lowest_free = 1;

arch/sparc/kernel/iommu.c

105

iommu->tbl.table_map_base = dma_offset;

arch/sparc/kernel/iommu.c

106

iommu->dma_addr_mask = dma_addr_mask;

arch/sparc/kernel/iommu.c

111

iommu->tbl.map = kzalloc_node(sz, GFP_KERNEL, numa_node);

arch/sparc/kernel/iommu.c

112

if (!iommu->tbl.map)

arch/sparc/kernel/iommu.c

115

iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,

arch/sparc/kernel/iommu.c

127

iommu->dummy_page = (unsigned long) page_address(page);

arch/sparc/kernel/iommu.c

128

memset((void *)iommu->dummy_page, 0, PAGE_SIZE);

arch/sparc/kernel/iommu.c

129

iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);

arch/sparc/kernel/iommu.c

138

iommu->page_table = (iopte_t *)page_address(page);

arch/sparc/kernel/iommu.c

141

iopte_make_dummy(iommu, &iommu->page_table[i]);

arch/sparc/kernel/iommu.c

146

free_page(iommu->dummy_page);

arch/sparc/kernel/iommu.c

147

iommu->dummy_page = 0UL;

arch/sparc/kernel/iommu.c

150

kfree(iommu->tbl.map);

arch/sparc/kernel/iommu.c

151

iommu->tbl.map = NULL;

arch/sparc/kernel/iommu.c

157

struct iommu *iommu,

arch/sparc/kernel/iommu.c

162

entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,

arch/sparc/kernel/iommu.c

167

return iommu->page_table + entry;

arch/sparc/kernel/iommu.c

170

static int iommu_alloc_ctx(struct iommu *iommu)

arch/sparc/kernel/iommu.c

172

int lowest = iommu->ctx_lowest_free;

arch/sparc/kernel/iommu.c

173

int n = find_next_zero_bit(iommu->ctx_bitmap, IOMMU_NUM_CTXS, lowest);

arch/sparc/kernel/iommu.c

176

n = find_next_zero_bit(iommu->ctx_bitmap, lowest, 1);

arch/sparc/kernel/iommu.c

183

__set_bit(n, iommu->ctx_bitmap);

arch/sparc/kernel/iommu.c

188

static inline void iommu_free_ctx(struct iommu *iommu, int ctx)

arch/sparc/kernel/iommu.c

191

__clear_bit(ctx, iommu->ctx_bitmap);

arch/sparc/kernel/iommu.c

192

if (ctx < iommu->ctx_lowest_free)

arch/sparc/kernel/iommu.c

193

iommu->ctx_lowest_free = ctx;

arch/sparc/kernel/iommu.c

202

struct iommu *iommu;

arch/sparc/kernel/iommu.c

221

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

223

iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT);

arch/sparc/kernel/iommu.c

230

*dma_addrp = (iommu->tbl.table_map_base +

arch/sparc/kernel/iommu.c

231

((iopte - iommu->page_table) << IO_PAGE_SHIFT));

arch/sparc/kernel/iommu.c

250

struct iommu *iommu;

arch/sparc/kernel/iommu.c

254

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

256

iommu_tbl_range_free(&iommu->tbl, dvma, npages, IOMMU_ERROR_CODE);

arch/sparc/kernel/iommu.c

267

struct iommu *iommu;

arch/sparc/kernel/iommu.c

285

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

295

base = alloc_npages(dev, iommu, npages);

arch/sparc/kernel/iommu.c

296

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

298

if (iommu->iommu_ctxflush)

arch/sparc/kernel/iommu.c

299

ctx = iommu_alloc_ctx(iommu);

arch/sparc/kernel/iommu.c

300

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

305

bus_addr = (iommu->tbl.table_map_base +

arch/sparc/kernel/iommu.c

306

((base - iommu->page_table) << IO_PAGE_SHIFT));

arch/sparc/kernel/iommu.c

323

iommu_free_ctx(iommu, ctx);

arch/sparc/kernel/iommu.c

330

static void strbuf_flush(struct strbuf *strbuf, struct iommu *iommu,

arch/sparc/kernel/iommu.c

337

iommu->iommu_ctxflush) {

arch/sparc/kernel/iommu.c

380

(void) iommu_read(iommu->write_complete_reg);

arch/sparc/kernel/iommu.c

400

struct iommu *iommu;

arch/sparc/kernel/iommu.c

411

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

416

base = iommu->page_table +

arch/sparc/kernel/iommu.c

417

((bus_addr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT);

arch/sparc/kernel/iommu.c

420

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

424

if (iommu->iommu_ctxflush)

arch/sparc/kernel/iommu.c

429

strbuf_flush(strbuf, iommu, bus_addr, ctx,

arch/sparc/kernel/iommu.c

434

iopte_make_dummy(iommu, base + i);

arch/sparc/kernel/iommu.c

436

iommu_free_ctx(iommu, ctx);

arch/sparc/kernel/iommu.c

437

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

439

iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE);

arch/sparc/kernel/iommu.c

453

struct iommu *iommu;

arch/sparc/kernel/iommu.c

458

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

460

if (nelems == 0 || !iommu)

arch/sparc/kernel/iommu.c

463

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

466

if (iommu->iommu_ctxflush)

arch/sparc/kernel/iommu.c

467

ctx = iommu_alloc_ctx(iommu);

arch/sparc/kernel/iommu.c

486

base_shift = iommu->tbl.table_map_base >> IO_PAGE_SHIFT;

arch/sparc/kernel/iommu.c

500

entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages,

arch/sparc/kernel/iommu.c

507

" npages %lx\n", iommu, paddr, npages);

arch/sparc/kernel/iommu.c

511

base = iommu->page_table + entry;

arch/sparc/kernel/iommu.c

514

dma_addr = iommu->tbl.table_map_base +

arch/sparc/kernel/iommu.c

52

struct iommu *iommu = container_of(iommu_map_table, struct iommu, tbl);

arch/sparc/kernel/iommu.c

53

if (iommu->iommu_flushinv) {

arch/sparc/kernel/iommu.c

54

iommu_write(iommu->iommu_flushinv, ~(u64)0);

arch/sparc/kernel/iommu.c

555

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

574

entry = (vaddr - iommu->tbl.table_map_base)

arch/sparc/kernel/iommu.c

576

base = iommu->page_table + entry;

arch/sparc/kernel/iommu.c

579

iopte_make_dummy(iommu, base + j);

arch/sparc/kernel/iommu.c

581

iommu_tbl_range_free(&iommu->tbl, vaddr, npages,

arch/sparc/kernel/iommu.c

589

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

59

tag = iommu->iommu_tags;

arch/sparc/kernel/iommu.c

597

static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg)

arch/sparc/kernel/iommu.c

601

if (iommu->iommu_ctxflush) {

arch/sparc/kernel/iommu.c

604

struct iommu_map_table *tbl = &iommu->tbl;

arch/sparc/kernel/iommu.c

607

base = iommu->page_table +

arch/sparc/kernel/iommu.c

622

struct iommu *iommu;

arch/sparc/kernel/iommu.c

626

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

629

ctx = fetch_sg_ctx(iommu, sglist);

arch/sparc/kernel/iommu.c

631

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

645

entry = ((dma_handle - iommu->tbl.table_map_base)

arch/sparc/kernel/iommu.c

647

base = iommu->page_table + entry;

arch/sparc/kernel/iommu.c

651

strbuf_flush(strbuf, iommu, dma_handle, ctx,

arch/sparc/kernel/iommu.c

655

iopte_make_dummy(iommu, base + i);

arch/sparc/kernel/iommu.c

657

iommu_tbl_range_free(&iommu->tbl, dma_handle, npages,

arch/sparc/kernel/iommu.c

66

(void) iommu_read(iommu->write_complete_reg);

arch/sparc/kernel/iommu.c

662

iommu_free_ctx(iommu, ctx);

arch/sparc/kernel/iommu.c

664

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

671

struct iommu *iommu;

arch/sparc/kernel/iommu.c

675

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

681

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

689

if (iommu->iommu_ctxflush &&

arch/sparc/kernel/iommu.c

692

struct iommu_map_table *tbl = &iommu->tbl;

arch/sparc/kernel/iommu.c

694

iopte = iommu->page_table +

arch/sparc/kernel/iommu.c

700

strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);

arch/sparc/kernel/iommu.c

702

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

709

struct iommu *iommu;

arch/sparc/kernel/iommu.c

715

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

721

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

725

if (iommu->iommu_ctxflush &&

arch/sparc/kernel/iommu.c

728

struct iommu_map_table *tbl = &iommu->tbl;

arch/sparc/kernel/iommu.c

730

iopte = iommu->page_table + ((sglist[0].dma_address -

arch/sparc/kernel/iommu.c

746

strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);

arch/sparc/kernel/iommu.c

748

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

753

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

758

if (device_mask < iommu->dma_addr_mask)

arch/sparc/kernel/iommu.c

80

#define IOPTE_IS_DUMMY(iommu, iopte) \

arch/sparc/kernel/iommu.c

81

((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)

arch/sparc/kernel/iommu.c

83

static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte)

arch/sparc/kernel/iommu.c

88

val |= iommu->dummy_page_pa;

arch/sparc/kernel/iommu.c

93

int iommu_table_init(struct iommu *iommu, int tsbsize,

arch/sparc/kernel/ldc.c

1022

static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,

arch/sparc/kernel/ldc.c

1029

base = iommu->page_table + entry;

arch/sparc/kernel/ldc.c

1044

struct ldc_iommu *ldc_iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

1045

struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;

arch/sparc/kernel/ldc.c

1056

iommu->map = kzalloc(sz, GFP_KERNEL);

arch/sparc/kernel/ldc.c

1057

if (!iommu->map) {

arch/sparc/kernel/ldc.c

1061

iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,

arch/sparc/kernel/ldc.c

1094

kfree(iommu->map);

arch/sparc/kernel/ldc.c

1095

iommu->map = NULL;

arch/sparc/kernel/ldc.c

1102

struct ldc_iommu *ldc_iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

1103

struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;

arch/sparc/kernel/ldc.c

1108

num_tsb_entries = iommu->poolsize * iommu->nr_pools;

arch/sparc/kernel/ldc.c

1115

kfree(iommu->map);

arch/sparc/kernel/ldc.c

1116

iommu->map = NULL;

arch/sparc/kernel/ldc.c

146

struct ldc_iommu iommu;

arch/sparc/kernel/ldc.c

2017

static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,

arch/sparc/kernel/ldc.c

2022

entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,

arch/sparc/kernel/ldc.c

2027

return iommu->page_table + entry;

arch/sparc/kernel/ldc.c

2155

struct ldc_iommu *iommu;

arch/sparc/kernel/ldc.c

2170

iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

2172

base = alloc_npages(iommu, npages);

arch/sparc/kernel/ldc.c

2177

state.page_table = iommu->page_table;

arch/sparc/kernel/ldc.c

2181

state.pte_idx = (base - iommu->page_table);

arch/sparc/kernel/ldc.c

2201

struct ldc_iommu *iommu;

arch/sparc/kernel/ldc.c

2212

iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

2214

base = alloc_npages(iommu, npages);

arch/sparc/kernel/ldc.c

2219

state.page_table = iommu->page_table;

arch/sparc/kernel/ldc.c

2223

state.pte_idx = (base - iommu->page_table);

arch/sparc/kernel/ldc.c

2233

static void free_npages(unsigned long id, struct ldc_iommu *iommu,

arch/sparc/kernel/ldc.c

2240

entry = ldc_cookie_to_index(cookie, iommu);

arch/sparc/kernel/ldc.c

2241

ldc_demap(iommu, id, cookie, entry, npages);

arch/sparc/kernel/ldc.c

2242

iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);

arch/sparc/kernel/ldc.c

2248

struct ldc_iommu *iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

2252

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/ldc.c

2257

free_npages(lp->id, iommu, addr, size);

arch/sparc/kernel/ldc.c

2259

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/of_device_common.c

66

op->dev.archdata.iommu = bus_sd->iommu;

arch/sparc/kernel/pci.c

275

static void pci_init_dev_archdata(struct dev_archdata *sd, void *iommu,

arch/sparc/kernel/pci.c

280

sd->iommu = iommu;

arch/sparc/kernel/pci.c

302

pci_init_dev_archdata(sd, pbm->iommu, &pbm->stc, pbm, op,

arch/sparc/kernel/pci.c

305

sd->iommu = pbm->iommu;

arch/sparc/kernel/pci.c

824

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/pci.c

841

if (iommu->dma_addr_mask & 0x80000000)

arch/sparc/kernel/pci.c

886

pci_init_dev_archdata(&dev->dev.archdata, psd->iommu,

arch/sparc/kernel/pci_fire.c

33

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_fire.c

45

iommu->iommu_control = pbm->pbm_regs + FIRE_IOMMU_CONTROL;

arch/sparc/kernel/pci_fire.c

46

iommu->iommu_tsbbase = pbm->pbm_regs + FIRE_IOMMU_TSBBASE;

arch/sparc/kernel/pci_fire.c

464

struct iommu *iommu;

arch/sparc/kernel/pci_fire.c

47

iommu->iommu_flush = pbm->pbm_regs + FIRE_IOMMU_FLUSH;

arch/sparc/kernel/pci_fire.c

477

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_fire.c

478

if (!iommu) {

arch/sparc/kernel/pci_fire.c

48

iommu->iommu_flushinv = pbm->pbm_regs + FIRE_IOMMU_FLUSHINV;

arch/sparc/kernel/pci_fire.c

483

pbm->iommu = iommu;

arch/sparc/kernel/pci_fire.c

494

kfree(pbm->iommu);

arch/sparc/kernel/pci_fire.c

53

iommu->write_complete_reg = pbm->controller_regs + 0x410000UL;

arch/sparc/kernel/pci_fire.c

58

upa_writeq(~(u64)0, iommu->iommu_flushinv);

arch/sparc/kernel/pci_fire.c

60

err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask,

arch/sparc/kernel/pci_fire.c

65

upa_writeq(__pa(iommu->page_table) | 0x7UL, iommu->iommu_tsbbase);

arch/sparc/kernel/pci_fire.c

67

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_fire.c

72

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_impl.h

150

struct iommu *iommu;

arch/sparc/kernel/pci_psycho.c

515

struct iommu *iommu;

arch/sparc/kernel/pci_psycho.c

530

iommu = pbm->sibling->iommu;

arch/sparc/kernel/pci_psycho.c

532

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_psycho.c

533

if (!iommu) {

arch/sparc/kernel/pci_psycho.c

539

pbm->iommu = iommu;

arch/sparc/kernel/pci_psycho.c

590

kfree(pbm->iommu);

arch/sparc/kernel/pci_sabre.c

466

struct iommu *iommu;

arch/sparc/kernel/pci_sabre.c

491

iommu = kzalloc_obj(*iommu);

arch/sparc/kernel/pci_sabre.c

492

if (!iommu) {

arch/sparc/kernel/pci_sabre.c

497

pbm->iommu = iommu;

arch/sparc/kernel/pci_sabre.c

580

kfree(pbm->iommu);

arch/sparc/kernel/pci_schizo.c

1140

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_schizo.c

1173

iommu->iommu_control = pbm->pbm_regs + SCHIZO_IOMMU_CONTROL;

arch/sparc/kernel/pci_schizo.c

1174

iommu->iommu_tsbbase = pbm->pbm_regs + SCHIZO_IOMMU_TSBBASE;

arch/sparc/kernel/pci_schizo.c

1175

iommu->iommu_flush = pbm->pbm_regs + SCHIZO_IOMMU_FLUSH;

arch/sparc/kernel/pci_schizo.c

1176

iommu->iommu_tags = iommu->iommu_flush + (0xa580UL - 0x0210UL);

arch/sparc/kernel/pci_schizo.c

1177

iommu->iommu_ctxflush = pbm->pbm_regs + SCHIZO_IOMMU_CTXFLUSH;

arch/sparc/kernel/pci_schizo.c

1182

iommu->write_complete_reg = pbm->controller_regs + 0x10000UL;

arch/sparc/kernel/pci_schizo.c

1187

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1189

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1201

err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask,

arch/sparc/kernel/pci_schizo.c

1208

upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);

arch/sparc/kernel/pci_schizo.c

1210

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1222

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1422

struct iommu *iommu;

arch/sparc/kernel/pci_schizo.c

1437

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_schizo.c

1438

if (!iommu) {

arch/sparc/kernel/pci_schizo.c

1443

pbm->iommu = iommu;

arch/sparc/kernel/pci_schizo.c

1456

kfree(pbm->iommu);

arch/sparc/kernel/pci_schizo.c

242

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_schizo.c

249

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/pci_schizo.c

250

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

257

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

288

iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

304

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

346

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/pci_sun4v.c

102

if (!iommu_use_atu(pbm->iommu, mask)) {

arch/sparc/kernel/pci_sun4v.c

119

iotsb_num = pbm->iommu->atu->iotsb->iotsb_num;

arch/sparc/kernel/pci_sun4v.c

1215

if (pbm->iommu->atu) {

arch/sparc/kernel/pci_sun4v.c

1218

kfree(pbm->iommu->atu);

arch/sparc/kernel/pci_sun4v.c

1219

pbm->iommu->atu = NULL;

arch/sparc/kernel/pci_sun4v.c

1236

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

1303

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_sun4v.c

1304

if (!iommu) {

arch/sparc/kernel/pci_sun4v.c

1309

pbm->iommu = iommu;

arch/sparc/kernel/pci_sun4v.c

1310

iommu->atu = NULL;

arch/sparc/kernel/pci_sun4v.c

1316

iommu->atu = atu;

arch/sparc/kernel/pci_sun4v.c

1328

kfree(iommu->atu);

arch/sparc/kernel/pci_sun4v.c

1329

kfree(pbm->iommu);

arch/sparc/kernel/pci_sun4v.c

188

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

213

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

215

if (!iommu_use_atu(iommu, mask))

arch/sparc/kernel/pci_sun4v.c

216

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

218

tbl = &iommu->atu->tbl;

arch/sparc/kernel/pci_sun4v.c

327

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

335

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

337

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

340

if (!iommu_use_atu(iommu, dvma)) {

arch/sparc/kernel/pci_sun4v.c

341

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

359

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

378

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

379

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

389

if (!iommu_use_atu(iommu, mask))

arch/sparc/kernel/pci_sun4v.c

390

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

443

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

457

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

459

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

468

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

488

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

497

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

498

if (nelems == 0 || !iommu)

arch/sparc/kernel/pci_sun4v.c

500

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

525

if (!iommu_use_atu(iommu, mask))

arch/sparc/kernel/pci_sun4v.c

526

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

639

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

647

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

649

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

668

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

686

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

690

if (device_mask < iommu->dma_addr_mask)

arch/sparc/kernel/pci_sun4v.c

719

struct iommu_map_table *iommu)

arch/sparc/kernel/pci_sun4v.c

726

for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) {

arch/sparc/kernel/pci_sun4v.c

727

pool = &(iommu->pools[pool_nr]);

arch/sparc/kernel/pci_sun4v.c

741

__set_bit(i, iommu->map);

arch/sparc/kernel/pci_sun4v.c

751

struct atu *atu = pbm->iommu->atu;

arch/sparc/kernel/pci_sun4v.c

78

static inline bool iommu_use_atu(struct iommu *iommu, u64 mask)

arch/sparc/kernel/pci_sun4v.c

80

return iommu->atu && mask > DMA_BIT_MASK(32);

arch/sparc/kernel/pci_sun4v.c

812

struct atu *atu = pbm->iommu->atu;

arch/sparc/kernel/pci_sun4v.c

878

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_sun4v.c

899

spin_lock_init(&iommu->lock);

arch/sparc/kernel/pci_sun4v.c

900

iommu->ctx_lowest_free = 1;

arch/sparc/kernel/pci_sun4v.c

901

iommu->tbl.table_map_base = dma_offset;

arch/sparc/kernel/pci_sun4v.c

902

iommu->dma_addr_mask = dma_mask;

arch/sparc/kernel/pci_sun4v.c

907

iommu->tbl.map = kzalloc(sz, GFP_KERNEL);

arch/sparc/kernel/pci_sun4v.c

908

if (!iommu->tbl.map) {

arch/sparc/kernel/pci_sun4v.c

912

iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,

arch/sparc/kernel/pci_sun4v.c

916

sz = probe_existing_entries(pbm, &iommu->tbl);

arch/sparc/kernel/psycho_common.c

209

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/psycho_common.c

212

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/psycho_common.c

213

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

218

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

248

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/psycho_common.c

405

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/psycho_common.c

409

iommu->iommu_control = pbm->controller_regs + PSYCHO_IOMMU_CONTROL;

arch/sparc/kernel/psycho_common.c

410

iommu->iommu_tsbbase = pbm->controller_regs + PSYCHO_IOMMU_TSBBASE;

arch/sparc/kernel/psycho_common.c

411

iommu->iommu_flush = pbm->controller_regs + PSYCHO_IOMMU_FLUSH;

arch/sparc/kernel/psycho_common.c

412

iommu->iommu_tags = pbm->controller_regs + PSYCHO_IOMMU_TAG;

arch/sparc/kernel/psycho_common.c

413

iommu->write_complete_reg = (pbm->controller_regs +

arch/sparc/kernel/psycho_common.c

416

iommu->iommu_ctxflush = 0;

arch/sparc/kernel/psycho_common.c

418

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

420

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

425

err = iommu_table_init(iommu, tsbsize * 1024 * 8,

arch/sparc/kernel/psycho_common.c

430

upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);

arch/sparc/kernel/psycho_common.c

432

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

447

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/sbus.c

213

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

214

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

275

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

276

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

349

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

350

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

428

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

433

reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

497

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

498

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

535

control = upa_readq(iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

537

upa_writeq(control, iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

545

struct iommu *iommu;

arch/sparc/kernel/sbus.c

559

iommu = kzalloc_obj(*iommu, GFP_ATOMIC);

arch/sparc/kernel/sbus.c

561

if (!iommu || !strbuf)

arch/sparc/kernel/sbus.c

564

op->dev.archdata.iommu = iommu;

arch/sparc/kernel/sbus.c

569

iommu->iommu_control = reg_base + IOMMU_CONTROL;

arch/sparc/kernel/sbus.c

570

iommu->iommu_tsbbase = reg_base + IOMMU_TSBBASE;

arch/sparc/kernel/sbus.c

571

iommu->iommu_flush = reg_base + IOMMU_FLUSH;

arch/sparc/kernel/sbus.c

572

iommu->iommu_tags = iommu->iommu_control +

arch/sparc/kernel/sbus.c

592

iommu->write_complete_reg = regs + 0x2000UL;

arch/sparc/kernel/sbus.c

599

if (iommu_table_init(iommu, IO_TSB_SIZE, MAP_BASE, 0xffffffff, -1))

arch/sparc/kernel/sbus.c

602

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/sbus.c

607

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/sbus.c

615

dram = iommu->iommu_control + (IOMMU_DRAMDIAG - IOMMU_CONTROL);

arch/sparc/kernel/sbus.c

616

tag = iommu->iommu_control + (IOMMU_TAGDIAG - IOMMU_CONTROL);

arch/sparc/kernel/sbus.c

623

upa_readq(iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

626

upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);

arch/sparc/kernel/sbus.c

63

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/sbus.c

648

control = upa_readq(iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

650

upa_writeq(control, iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

660

kfree(iommu);

arch/sparc/kernel/sbus.c

78

cfg_reg = iommu->write_complete_reg;

arch/sparc/mm/io-unit.c

150

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

167

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

186

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

201

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

221

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

66

op->dev.archdata.iommu = iounit;

arch/sparc/mm/iommu.c

102

iommu->page_table = (iopte_t *)tmp;

arch/sparc/mm/iommu.c

105

memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));

arch/sparc/mm/iommu.c

109

base = __pa((unsigned long)iommu->page_table) >> 4;

arch/sparc/mm/iommu.c

110

sbus_writel(base, &iommu->regs->base);

arch/sparc/mm/iommu.c

111

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

119

bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);

arch/sparc/mm/iommu.c

124

iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;

arch/sparc/mm/iommu.c

126

iommu->usemap.num_colors = 1;

arch/sparc/mm/iommu.c

129

impl, vers, iommu->page_table,

arch/sparc/mm/iommu.c

132

op->dev.archdata.iommu = iommu;

arch/sparc/mm/iommu.c

187

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

216

ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);

arch/sparc/mm/iommu.c

219

busa0 = iommu->start + (ioptex << PAGE_SHIFT);

arch/sparc/mm/iommu.c

220

iopte0 = &iommu->page_table[ioptex];

arch/sparc/mm/iommu.c

226

iommu_invalidate_page(iommu->regs, busa);

arch/sparc/mm/iommu.c

285

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

289

unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;

arch/sparc/mm/iommu.c

292

BUG_ON(busa < iommu->start);

arch/sparc/mm/iommu.c

294

iopte_val(iommu->page_table[ioptex + i]) = 0;

arch/sparc/mm/iommu.c

295

iommu_invalidate_page(iommu->regs, busa);

arch/sparc/mm/iommu.c

298

bit_map_clear(&iommu->usemap, ioptex, npages);

arch/sparc/mm/iommu.c

318

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

320

iopte_t *iopte = iommu->page_table;

arch/sparc/mm/iommu.c

342

ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,

arch/sparc/mm/iommu.c

387

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

389

*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);

arch/sparc/mm/iommu.c

400

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

401

iopte_t *iopte = iommu->page_table;

arch/sparc/mm/iommu.c

403

int ioptex = (busa - iommu->start) >> PAGE_SHIFT;

arch/sparc/mm/iommu.c

419

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

420

bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);

arch/sparc/mm/iommu.c

60

struct iommu_struct *iommu;

arch/sparc/mm/iommu.c

67

iommu = kmalloc_obj(struct iommu_struct);

arch/sparc/mm/iommu.c

68

if (!iommu) {

arch/sparc/mm/iommu.c

73

iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,

arch/sparc/mm/iommu.c

75

if (!iommu->regs) {

arch/sparc/mm/iommu.c

80

control = sbus_readl(&iommu->regs->control);

arch/sparc/mm/iommu.c

85

sbus_writel(control, &iommu->regs->control);

arch/sparc/mm/iommu.c

87

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

88

iommu->start = IOMMU_START;

arch/sparc/mm/iommu.c

89

iommu->end = 0xffffffff;

arch/x86/events/amd/iommu.c

236

return (container_of(ev->pmu, struct perf_amd_iommu, pmu))->iommu;

arch/x86/events/amd/iommu.c

241

struct amd_iommu *iommu = perf_event_2_iommu(ev);

arch/x86/events/amd/iommu.c

248

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_COUNTER_SRC_REG, &reg);

arch/x86/events/amd/iommu.c

254

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DEVID_MATCH_REG, &reg);

arch/x86/events/amd/iommu.c

260

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_PASID_MATCH_REG, &reg);

arch/x86/events/amd/iommu.c

266

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DOMID_MATCH_REG, &reg);

arch/x86/events/amd/iommu.c

271

struct amd_iommu *iommu = perf_event_2_iommu(event);

arch/x86/events/amd/iommu.c

275

amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,

arch/x86/events/amd/iommu.c

298

struct amd_iommu *iommu = perf_event_2_iommu(event);

arch/x86/events/amd/iommu.c

304

amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,

arch/x86/events/amd/iommu.c

315

struct amd_iommu *iommu = perf_event_2_iommu(event);

arch/x86/events/amd/iommu.c

317

if (amd_iommu_pc_get_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,

arch/x86/events/amd/iommu.c

40

struct amd_iommu *iommu;

arch/x86/events/amd/iommu.c

432

perf_iommu->iommu = get_amd_iommu(idx);

arch/x86/events/amd/iommu.c

436

if (!perf_iommu->iommu ||

arch/x86/include/asm/pci.h

21

void *iommu; /* IOMMU private data */

arch/x86/include/asm/x86_init.h

181

struct x86_init_iommu iommu;

arch/x86/kernel/aperture_64.c

431

x86_init.iommu.iommu_init = gart_iommu_init;

arch/x86/kernel/pci-dma.c

174

x86_init.iommu.iommu_init();

arch/x86/kernel/x86_init.c

104

.iommu = {

drivers/acpi/arm64/iort.c

1101

struct acpi_iort_node *iommu,

drivers/acpi/arm64/iort.c

1135

if (parent != iommu)

drivers/acpi/arm64/iort.c

1159

static void iort_find_rmrs(struct acpi_iort_node *iommu, struct device *dev,

drivers/acpi/arm64/iort.c

1183

iort_node_get_rmr_info(iort_node, iommu, dev, head);

drivers/acpi/arm64/iort.c

1199

struct acpi_iort_node *iommu;

drivers/acpi/arm64/iort.c

1201

iommu = iort_get_iort_node(iommu_fwnode);

drivers/acpi/arm64/iort.c

1202

if (!iommu)

drivers/acpi/arm64/iort.c

1205

iort_find_rmrs(iommu, dev, head);

drivers/acpi/arm64/iort.c

1210

struct acpi_iort_node *iommu;

drivers/acpi/arm64/iort.c

1213

iommu = iort_get_iort_node(fwspec->iommu_fwnode);

drivers/acpi/arm64/iort.c

1215

if (iommu && (iommu->type == ACPI_IORT_NODE_SMMU_V3)) {

drivers/acpi/arm64/iort.c

1218

smmu = (struct acpi_iort_smmu_v3 *)iommu->node_data;

drivers/acpi/arm64/iort.c

1220

return iommu;

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

250

mdev->iommu = iommu_get_domain_for_dev(mdev->dev);

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

251

if (!mdev->iommu)

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

320

if (mdev->iommu && mdev->funcs->connect_iommu)

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

329

if (mdev->iommu && mdev->funcs->disconnect_iommu)

drivers/gpu/drm/arm/display/komeda/komeda_dev.h

198

struct iommu_domain *iommu;

drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c

198

kfb->is_va = mdev->iommu ? true : false;

drivers/gpu/drm/i915/i915_gpu_error.c

2069

error->iommu = i915_vtd_active(i915);

drivers/gpu/drm/i915/i915_gpu_error.c

932

err_printf(m, "IOMMU enabled?: %d\n", error->iommu);

drivers/gpu/drm/i915/i915_gpu_error.h

205

int iommu;

drivers/gpu/drm/i915/selftests/mock_gem_device.c

162

pdev->dev.iommu = &fake_iommu;

drivers/gpu/drm/msm/msm_iommu.c

138

struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);

drivers/gpu/drm/msm/msm_iommu.c

139

phys_addr_t phys = page_to_phys(iommu->prr_page);

drivers/gpu/drm/msm/msm_iommu.c

224

struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);

drivers/gpu/drm/msm/msm_iommu.c

232

mutex_lock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

233

if (--iommu->pagetables == 0) {

drivers/gpu/drm/msm/msm_iommu.c

238

__free_page(iommu->prr_page);

drivers/gpu/drm/msm/msm_iommu.c

239

iommu->prr_page = NULL;

drivers/gpu/drm/msm/msm_iommu.c

242

mutex_unlock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

269

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

271

return &iommu->domain->geometry;

drivers/gpu/drm/msm/msm_iommu.c

507

struct msm_iommu *iommu = to_msm_iommu(parent);

drivers/gpu/drm/msm/msm_iommu.c

580

mutex_lock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

581

if (iommu->pagetables++ == 0) {

drivers/gpu/drm/msm/msm_iommu.c

584

iommu->pagetables--;

drivers/gpu/drm/msm/msm_iommu.c

585

mutex_unlock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

591

BUG_ON(iommu->prr_page);

drivers/gpu/drm/msm/msm_iommu.c

604

iommu->prr_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

drivers/gpu/drm/msm/msm_iommu.c

606

page_to_phys(iommu->prr_page));

drivers/gpu/drm/msm/msm_iommu.c

610

mutex_unlock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

632

struct msm_iommu *iommu = arg;

drivers/gpu/drm/msm/msm_iommu.c

633

struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(iommu->base.dev);

drivers/gpu/drm/msm/msm_iommu.c

641

if (iommu->base.handler)

drivers/gpu/drm/msm/msm_iommu.c

642

return iommu->base.handler(iommu->base.arg, iova, flags, ptr);

drivers/gpu/drm/msm/msm_iommu.c

652

struct msm_iommu *iommu = arg;

drivers/gpu/drm/msm/msm_iommu.c

654

if (iommu->base.handler)

drivers/gpu/drm/msm/msm_iommu.c

655

return iommu->base.handler(iommu->base.arg, iova, flags, NULL);

drivers/gpu/drm/msm/msm_iommu.c

670

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

672

iommu_detach_device(iommu->domain, mmu->dev);

drivers/gpu/drm/msm/msm_iommu.c

679

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

688

ret = iommu_map_sgtable(iommu->domain, iova, sgt, prot);

drivers/gpu/drm/msm/msm_iommu.c

696

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

701

iommu_unmap(iommu->domain, iova, len);

drivers/gpu/drm/msm/msm_iommu.c

708

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

709

iommu_domain_free(iommu->domain);

drivers/gpu/drm/msm/msm_iommu.c

710

kmem_cache_destroy(iommu->pt_cache);

drivers/gpu/drm/msm/msm_iommu.c

711

kfree(iommu);

drivers/gpu/drm/msm/msm_iommu.c

725

struct msm_iommu *iommu;

drivers/gpu/drm/msm/msm_iommu.c

737

iommu = kzalloc_obj(*iommu);

drivers/gpu/drm/msm/msm_iommu.c

738

if (!iommu) {

drivers/gpu/drm/msm/msm_iommu.c

743

iommu->domain = domain;

drivers/gpu/drm/msm/msm_iommu.c

744

msm_mmu_init(&iommu->base, dev, &funcs, MSM_MMU_IOMMU);

drivers/gpu/drm/msm/msm_iommu.c

746

mutex_init(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

748

ret = iommu_attach_device(iommu->domain, dev);

drivers/gpu/drm/msm/msm_iommu.c

751

kfree(iommu);

drivers/gpu/drm/msm/msm_iommu.c

755

return &iommu->base;

drivers/gpu/drm/msm/msm_iommu.c

760

struct msm_iommu *iommu;

drivers/gpu/drm/msm/msm_iommu.c

767

iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

768

iommu_set_fault_handler(iommu->domain, msm_disp_fault_handler, iommu);

drivers/gpu/drm/msm/msm_iommu.c

776

struct msm_iommu *iommu;

drivers/gpu/drm/msm/msm_iommu.c

783

iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

789

iommu->pt_cache =

drivers/gpu/drm/msm/msm_iommu.c

792

iommu_set_fault_handler(iommu->domain, msm_gpu_fault_handler, iommu);

drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h

30

} iommu;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

120

mutex_init(&tdev->iommu.mutex);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

123

tdev->iommu.domain = iommu_paging_domain_alloc(dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

124

if (IS_ERR(tdev->iommu.domain))

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

132

pgsize_bitmap = tdev->iommu.domain->pgsize_bitmap;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

134

tdev->iommu.pgshift = PAGE_SHIFT;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

136

tdev->iommu.pgshift = fls(pgsize_bitmap & ~PAGE_MASK);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

137

if (tdev->iommu.pgshift == 0) {

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

141

tdev->iommu.pgshift -= 1;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

144

ret = iommu_attach_device(tdev->iommu.domain, dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

148

ret = nvkm_mm_init(&tdev->iommu.mm, 0, 0,

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

150

tdev->iommu.pgshift, 1);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

158

iommu_detach_device(tdev->iommu.domain, dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

161

iommu_domain_free(tdev->iommu.domain);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

164

tdev->iommu.domain = NULL;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

165

tdev->iommu.pgshift = 0;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

174

if (tdev->iommu.domain) {

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

175

nvkm_mm_fini(&tdev->iommu.mm);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

176

iommu_detach_device(tdev->iommu.domain, tdev->device.dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

177

iommu_domain_free(tdev->iommu.domain);

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

591

if (tdev->iommu.domain) {

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

592

imem->mm_mutex = &tdev->iommu.mutex;

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

593

imem->mm = &tdev->iommu.mm;

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

594

imem->domain = tdev->iommu.domain;

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

595

imem->iommu_pgshift = tdev->iommu.pgshift;

drivers/gpu/host1x/context.c

135

if (cd->dev.iommu->iommu_dev != dev->iommu->iommu_dev)

drivers/iommu/amd/amd_iommu.h

158

return iommu_get_iommu_dev(dev, struct amd_iommu, iommu);

drivers/iommu/amd/amd_iommu.h

164

return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu);

drivers/iommu/amd/amd_iommu.h

172

bool translation_pre_enabled(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

18

void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,

drivers/iommu/amd/amd_iommu.h

189

struct dev_table_entry *get_dev_table(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

190

struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/amd_iommu.h

196

void amd_iommu_update_dte(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

204

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/amd_iommu.h

21

void amd_iommu_restart_event_logging(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

214

initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);

drivers/iommu/amd/amd_iommu.h

22

void amd_iommu_restart_ga_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

23

void amd_iommu_restart_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

24

void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/amd_iommu.h

25

void iommu_feature_enable(struct amd_iommu *iommu, u8 bit);

drivers/iommu/amd/amd_iommu.h

26

void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

64

int amd_iommu_iopf_init(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

65

void amd_iommu_iopf_uninit(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

68

int amd_iommu_iopf_add_device(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

70

void amd_iommu_iopf_remove_device(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

79

int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

80

void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

81

void amd_iommu_enable_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

82

void amd_iommu_poll_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

89

void amd_iommu_flush_all_caches(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

96

int amd_iommu_create_irq_domain(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

98

static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu)

drivers/iommu/amd/amd_iommu_types.h

1038

struct amd_iommu *iommu;

drivers/iommu/amd/amd_iommu_types.h

1055

void (*activate)(struct amd_iommu *iommu, void *, u16, u16);

drivers/iommu/amd/amd_iommu_types.h

1056

void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);

drivers/iommu/amd/amd_iommu_types.h

1057

void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);

drivers/iommu/amd/amd_iommu_types.h

455

#define for_each_iommu(iommu) \

drivers/iommu/amd/amd_iommu_types.h

456

list_for_each_entry((iommu), &amd_iommu_list, list)

drivers/iommu/amd/amd_iommu_types.h

457

#define for_each_iommu_safe(iommu, next) \

drivers/iommu/amd/amd_iommu_types.h

458

list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)

drivers/iommu/amd/amd_iommu_types.h

500

struct amd_iommu *iommu; /* IOMMUs attach to protection domain */

drivers/iommu/amd/amd_iommu_types.h

543

struct pt_iommu iommu;

drivers/iommu/amd/amd_iommu_types.h

563

PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);

drivers/iommu/amd/amd_iommu_types.h

564

PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);

drivers/iommu/amd/amd_iommu_types.h

565

PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv2.iommu, domain);

drivers/iommu/amd/amd_iommu_types.h

724

struct iommu_device iommu;

drivers/iommu/amd/amd_iommu_types.h

776

struct iommu_device *iommu = dev_to_iommu_device(dev);

drivers/iommu/amd/amd_iommu_types.h

778

return container_of(iommu, struct amd_iommu, iommu);

drivers/iommu/amd/debugfs.c

101

err = pci_read_config_dword(iommu->dev, iommu->cap_ptr + iommu->dbg_cap_offset, &value);

drivers/iommu/amd/debugfs.c

104

iommu->dbg_cap_offset);

drivers/iommu/amd/debugfs.c

108

seq_printf(m, "Offset:0x%x Value:0x%08x\n", iommu->dbg_cap_offset, value);

drivers/iommu/amd/debugfs.c

116

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

122

raw_spin_lock_irqsave(&iommu->lock, flag);

drivers/iommu/amd/debugfs.c

123

head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

drivers/iommu/amd/debugfs.c

124

tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/debugfs.c

128

cmd = (struct iommu_cmd *)(iommu->cmd_buf + i * sizeof(*cmd));

drivers/iommu/amd/debugfs.c

132

raw_spin_unlock_irqrestore(&iommu->lock, flag);

drivers/iommu/amd/debugfs.c

143

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

177

iommu = pci_seg->rlookup_table[devid];

drivers/iommu/amd/debugfs.c

178

if (!iommu) {

drivers/iommu/amd/debugfs.c

215

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

217

iommu = pci_seg->rlookup_table[devid];

drivers/iommu/amd/debugfs.c

218

if (!iommu)

drivers/iommu/amd/debugfs.c

221

dev_table = get_dev_table(iommu);

drivers/iommu/amd/debugfs.c

233

seq_printf(m, "iommu%d\n", iommu->index);

drivers/iommu/amd/debugfs.c

28

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

296

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

307

iommu = pci_seg->rlookup_table[devid];

drivers/iommu/amd/debugfs.c

308

if (!iommu)

drivers/iommu/amd/debugfs.c

31

iommu->dbg_mmio_offset = -1;

drivers/iommu/amd/debugfs.c

311

dev_table = get_dev_table(iommu);

drivers/iommu/amd/debugfs.c

36

ret = kstrtou32_from_user(ubuf, cnt, 0, &iommu->dbg_mmio_offset);

drivers/iommu/amd/debugfs.c

366

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

371

for_each_iommu(iommu) {

drivers/iommu/amd/debugfs.c

372

iommu->dbg_mmio_offset = -1;

drivers/iommu/amd/debugfs.c

373

iommu->dbg_cap_offset = -1;

drivers/iommu/amd/debugfs.c

375

snprintf(name, MAX_NAME_LEN, "iommu%02d", iommu->index);

drivers/iommu/amd/debugfs.c

376

iommu->debugfs = debugfs_create_dir(name, amd_iommu_debugfs);

drivers/iommu/amd/debugfs.c

378

debugfs_create_file("mmio", 0644, iommu->debugfs, iommu,

drivers/iommu/amd/debugfs.c

380

debugfs_create_file("capability", 0644, iommu->debugfs, iommu,

drivers/iommu/amd/debugfs.c

382

debugfs_create_file("cmdbuf", 0444, iommu->debugfs, iommu,

drivers/iommu/amd/debugfs.c

40

if (iommu->dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) {

drivers/iommu/amd/debugfs.c

41

iommu->dbg_mmio_offset = -1;

drivers/iommu/amd/debugfs.c

50

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

53

if (iommu->dbg_mmio_offset < 0) {

drivers/iommu/amd/debugfs.c

58

value = readq(iommu->mmio_base + iommu->dbg_mmio_offset);

drivers/iommu/amd/debugfs.c

59

seq_printf(m, "Offset:0x%x Value:0x%016llx\n", iommu->dbg_mmio_offset, value);

drivers/iommu/amd/debugfs.c

69

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

72

iommu->dbg_cap_offset = -1;

drivers/iommu/amd/debugfs.c

77

ret = kstrtou32_from_user(ubuf, cnt, 0, &iommu->dbg_cap_offset);

drivers/iommu/amd/debugfs.c

82

if (iommu->dbg_cap_offset > 0x14) {

drivers/iommu/amd/debugfs.c

83

iommu->dbg_cap_offset = -1;

drivers/iommu/amd/debugfs.c

92

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

96

if (iommu->dbg_cap_offset < 0) {

drivers/iommu/amd/init.c

1000

paddr = readq(iommu->mmio_base + MMIO_CMD_BUF_OFFSET) & PM_ADDR_MASK;

drivers/iommu/amd/init.c

1001

iommu->cmd_buf = iommu_memremap(paddr, CMD_BUFFER_SIZE);

drivers/iommu/amd/init.c

1003

return iommu->cmd_buf ? 0 : -ENOMEM;

drivers/iommu/amd/init.c

1006

static int __init remap_or_alloc_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1016

paddr = readq(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET) & PM_ADDR_MASK;

drivers/iommu/amd/init.c

1017

iommu->cmd_sem = iommu_memremap(paddr, PAGE_SIZE);

drivers/iommu/amd/init.c

1018

if (!iommu->cmd_sem)

drivers/iommu/amd/init.c

1020

iommu->cmd_sem_paddr = paddr;

drivers/iommu/amd/init.c

1022

return alloc_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1028

static int __init alloc_iommu_buffers(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1037

ret = remap_or_alloc_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1041

ret = remap_command_buffer(iommu);

drivers/iommu/amd/init.c

1045

ret = remap_event_buffer(iommu);

drivers/iommu/amd/init.c

1049

ret = alloc_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1053

ret = alloc_command_buffer(iommu);

drivers/iommu/amd/init.c

1057

ret = alloc_event_buffer(iommu);

drivers/iommu/amd/init.c

1065

static void __init free_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1067

if (iommu->cmd_sem)

drivers/iommu/amd/init.c

1068

iommu_free_pages((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

1070

static void __init unmap_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1072

if (iommu->cmd_sem) {

drivers/iommu/amd/init.c

1074

memunmap((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

1076

iommu_free_pages((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

1080

static void __init unmap_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1082

memunmap((void *)iommu->cmd_buf);

drivers/iommu/amd/init.c

1085

static void __init unmap_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1087

memunmap(iommu->evt_buf);

drivers/iommu/amd/init.c

1090

static void __init free_iommu_buffers(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1093

unmap_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1094

unmap_command_buffer(iommu);

drivers/iommu/amd/init.c

1095

unmap_event_buffer(iommu);

drivers/iommu/amd/init.c

1097

free_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1098

free_command_buffer(iommu);

drivers/iommu/amd/init.c

1099

free_event_buffer(iommu);

drivers/iommu/amd/init.c

1103

static void iommu_enable_xt(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1112

iommu_feature_enable(iommu, CONTROL_XT_EN);

drivers/iommu/amd/init.c

1116

static void iommu_enable_gt(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1121

iommu_feature_enable(iommu, CONTROL_GT_EN);

drivers/iommu/amd/init.c

1129

iommu_feature_enable(iommu, CONTROL_GCR3TRPMODE);

drivers/iommu/amd/init.c

1141

static bool __reuse_device_table(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1143

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/init.c

1152

lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);

drivers/iommu/amd/init.c

1153

hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);

drivers/iommu/amd/init.c

1159

iommu->index);

drivers/iommu/amd/init.c

1205

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

1218

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

1219

if (pci_seg->id != iommu->pci_seg->id)

drivers/iommu/amd/init.c

1221

if (!__reuse_device_table(iommu))

drivers/iommu/amd/init.c

1272

set_dev_entry_from_acpi_range(struct amd_iommu *iommu, u16 first, u16 last,

drivers/iommu/amd/init.c

1282

if (search_ivhd_dte_flags(iommu->pci_seg->id, first, last))

drivers/iommu/amd/init.c

1311

d->segid = iommu->pci_seg->id;

drivers/iommu/amd/init.c

1319

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/init.c

1323

amd_iommu_set_rlookup_table(iommu, i);

drivers/iommu/amd/init.c

1327

static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,

drivers/iommu/amd/init.c

1330

set_dev_entry_from_acpi_range(iommu, devid, devid, flags, ext_flags);

drivers/iommu/amd/init.c

1444

static int __init init_iommu_from_acpi(struct amd_iommu *iommu,

drivers/iommu/amd/init.c

1453

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/init.c

1467

iommu->acpi_flags = h->flags;

drivers/iommu/amd/init.c

1491

set_dev_entry_from_acpi_range(iommu, 0, pci_seg->last_bdf, e->flags, 0);

drivers/iommu/amd/init.c

1502

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

drivers/iommu/amd/init.c

1530

set_dev_entry_from_acpi(iommu, devid , e->flags, 0);

drivers/iommu/amd/init.c

1531

set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);

drivers/iommu/amd/init.c

1560

set_dev_entry_from_acpi(iommu, devid, e->flags,

drivers/iommu/amd/init.c

1587

set_dev_entry_from_acpi(iommu, devid_to, flags, ext_flags);

drivers/iommu/amd/init.c

1589

set_dev_entry_from_acpi_range(iommu, devid_start, devid, flags, ext_flags);

drivers/iommu/amd/init.c

1624

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

drivers/iommu/amd/init.c

1691

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

drivers/iommu/amd/init.c

1782

static void __init free_sysfs(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1784

if (iommu->iommu.dev) {

drivers/iommu/amd/init.c

1785

iommu_device_unregister(&iommu->iommu);

drivers/iommu/amd/init.c

1786

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/amd/init.c

1790

static void __init free_iommu_one(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1792

free_sysfs(iommu);

drivers/iommu/amd/init.c

1793

free_iommu_buffers(iommu);

drivers/iommu/amd/init.c

1794

amd_iommu_free_ppr_log(iommu);

drivers/iommu/amd/init.c

1795

free_ga_log(iommu);

drivers/iommu/amd/init.c

1796

iommu_unmap_mmio_space(iommu);

drivers/iommu/amd/init.c

1797

amd_iommu_iopf_uninit(iommu);

drivers/iommu/amd/init.c

1802

struct amd_iommu *iommu, *next;

drivers/iommu/amd/init.c

1804

for_each_iommu_safe(iommu, next) {

drivers/iommu/amd/init.c

1805

list_del(&iommu->list);

drivers/iommu/amd/init.c

1806

free_iommu_one(iommu);

drivers/iommu/amd/init.c

1807

kfree(iommu);

drivers/iommu/amd/init.c

1817

static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1826

pci_write_config_dword(iommu->dev, 0xf0, 0x90);

drivers/iommu/amd/init.c

1827

pci_read_config_dword(iommu->dev, 0xf4, &value);

drivers/iommu/amd/init.c

1833

pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));

drivers/iommu/amd/init.c

1835

pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);

drivers/iommu/amd/init.c

1836

pci_info(iommu->dev, "Applying erratum 746 workaround\n");

drivers/iommu/amd/init.c

1839

pci_write_config_dword(iommu->dev, 0xf0, 0x90);

drivers/iommu/amd/init.c

1848

static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1858

value = iommu_read_l2(iommu, 0x47);

drivers/iommu/amd/init.c

1864

iommu_write_l2(iommu, 0x47, value | BIT(0));

drivers/iommu/amd/init.c

1866

pci_info(iommu->dev, "Applying ATS write check workaround\n");

drivers/iommu/amd/init.c

1874

static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,

drivers/iommu/amd/init.c

1882

iommu->pci_seg = pci_seg;

drivers/iommu/amd/init.c

1884

raw_spin_lock_init(&iommu->lock);

drivers/iommu/amd/init.c

1885

iommu->cmd_sem_val = 0;

drivers/iommu/amd/init.c

1888

list_add_tail(&iommu->list, &amd_iommu_list);

drivers/iommu/amd/init.c

1889

iommu->index = amd_iommus_present++;

drivers/iommu/amd/init.c

1891

if (unlikely(iommu->index >= MAX_IOMMUS)) {

drivers/iommu/amd/init.c

1899

iommu->devid = h->devid;

drivers/iommu/amd/init.c

1900

iommu->cap_ptr = h->cap_ptr;

drivers/iommu/amd/init.c

1901

iommu->mmio_phys = h->mmio_phys;

drivers/iommu/amd/init.c

1909

iommu->mmio_phys_end = MMIO_REG_END_OFFSET;

drivers/iommu/amd/init.c

1911

iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;

drivers/iommu/amd/init.c

1920

iommu->mmio_phys_end = MMIO_REG_END_OFFSET;

drivers/iommu/amd/init.c

1922

iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;

drivers/iommu/amd/init.c

1938

early_iommu_features_init(iommu, h);

drivers/iommu/amd/init.c

1945

iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,

drivers/iommu/amd/init.c

1946

iommu->mmio_phys_end);

drivers/iommu/amd/init.c

1947

if (!iommu->mmio_base)

drivers/iommu/amd/init.c

1950

return init_iommu_from_acpi(iommu, h);

drivers/iommu/amd/init.c

1953

static int __init init_iommu_one_late(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1957

ret = alloc_iommu_buffers(iommu);

drivers/iommu/amd/init.c

1961

iommu->int_enabled = false;

drivers/iommu/amd/init.c

1963

init_translation_status(iommu);

drivers/iommu/amd/init.c

1964

if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {

drivers/iommu/amd/init.c

1965

iommu_disable(iommu);

drivers/iommu/amd/init.c

1966

clear_translation_pre_enabled(iommu);

drivers/iommu/amd/init.c

1968

iommu->index);

drivers/iommu/amd/init.c

1971

amd_iommu_pre_enabled = translation_pre_enabled(iommu);

drivers/iommu/amd/init.c

1974

ret = amd_iommu_create_irq_domain(iommu);

drivers/iommu/amd/init.c

1983

iommu->pci_seg->rlookup_table[iommu->devid] = NULL;

drivers/iommu/amd/init.c

2022

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2041

iommu = kzalloc_obj(struct amd_iommu);

drivers/iommu/amd/init.c

2042

if (iommu == NULL)

drivers/iommu/amd/init.c

2045

ret = init_iommu_one(iommu, h, table);

drivers/iommu/amd/init.c

2058

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2059

ret = init_iommu_one_late(iommu);

drivers/iommu/amd/init.c

2067

static void init_iommu_perf_ctr(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2070

struct pci_dev *pdev = iommu->dev;

drivers/iommu/amd/init.c

2079

val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);

drivers/iommu/amd/init.c

2080

iommu->max_banks = (u8) ((val >> 12) & 0x3f);

drivers/iommu/amd/init.c

2081

iommu->max_counters = (u8) ((val >> 7) & 0xf);

drivers/iommu/amd/init.c

2090

struct amd_iommu *iommu = dev_to_amd_iommu(dev);

drivers/iommu/amd/init.c

2091

return sysfs_emit(buf, "%x\n", iommu->cap);

drivers/iommu/amd/init.c

2124

static void __init late_iommu_features_init(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2128

if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))

drivers/iommu/amd/init.c

2132

features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);

drivers/iommu/amd/init.c

2133

features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);

drivers/iommu/amd/init.c

2154

static int __init iommu_init_pci(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2156

int cap_ptr = iommu->cap_ptr;

drivers/iommu/amd/init.c

2159

iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,

drivers/iommu/amd/init.c

2160

PCI_BUS_NUM(iommu->devid),

drivers/iommu/amd/init.c

2161

iommu->devid & 0xff);

drivers/iommu/amd/init.c

2162

if (!iommu->dev)

drivers/iommu/amd/init.c

2166

iommu->dev->irq_managed = 1;

drivers/iommu/amd/init.c

2168

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,

drivers/iommu/amd/init.c

2169

&iommu->cap);

drivers/iommu/amd/init.c

2171

if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))

drivers/iommu/amd/init.c

2174

late_iommu_features_init(iommu);

drivers/iommu/amd/init.c

2181

iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1;

drivers/iommu/amd/init.c

2183

BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK);

drivers/iommu/amd/init.c

2192

iommu_enable_gt(iommu);

drivers/iommu/amd/init.c

2195

if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))

drivers/iommu/amd/init.c

2198

if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {

drivers/iommu/amd/init.c

2204

init_iommu_perf_ctr(iommu);

drivers/iommu/amd/init.c

2206

if (is_rd890_iommu(iommu->dev)) {

drivers/iommu/amd/init.c

2209

iommu->root_pdev =

drivers/iommu/amd/init.c

2210

pci_get_domain_bus_and_slot(iommu->pci_seg->id,

drivers/iommu/amd/init.c

2211

iommu->dev->bus->number,

drivers/iommu/amd/init.c

2219

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,

drivers/iommu/amd/init.c

2220

&iommu->stored_addr_lo);

drivers/iommu/amd/init.c

2221

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,

drivers/iommu/amd/init.c

2222

&iommu->stored_addr_hi);

drivers/iommu/amd/init.c

2225

iommu->stored_addr_lo &= ~1;

drivers/iommu/amd/init.c

2229

iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);

drivers/iommu/amd/init.c

2232

iommu->stored_l2[i] = iommu_read_l2(iommu, i);

drivers/iommu/amd/init.c

2235

amd_iommu_erratum_746_workaround(iommu);

drivers/iommu/amd/init.c

2236

amd_iommu_ats_write_check_workaround(iommu);

drivers/iommu/amd/init.c

2238

ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,

drivers/iommu/amd/init.c

2239

amd_iommu_groups, "ivhd%d", iommu->index);

drivers/iommu/amd/init.c

2248

ret = amd_iommu_iopf_init(iommu);

drivers/iommu/amd/init.c

2253

ret = iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);

drivers/iommu/amd/init.c

2260

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/amd/init.c

2263

return pci_enable_device(iommu->dev);

drivers/iommu/amd/init.c

2307

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2314

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2315

ret = iommu_init_pci(iommu);

drivers/iommu/amd/init.c

2318

iommu->index, ret);

drivers/iommu/amd/init.c

232

bool translation_pre_enabled(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2322

iommu_set_cwwb_range(iommu);

drivers/iommu/amd/init.c

2338

for_each_iommu(iommu)

drivers/iommu/amd/init.c

2339

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

234

return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);

drivers/iommu/amd/init.c

2356

static int iommu_setup_msi(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2360

r = pci_enable_msi(iommu->dev);

drivers/iommu/amd/init.c

2364

r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread,

drivers/iommu/amd/init.c

2365

IRQF_ONESHOT, "AMD-Vi", iommu);

drivers/iommu/amd/init.c

2367

pci_disable_msi(iommu->dev);

drivers/iommu/amd/init.c

237

static void clear_translation_pre_enabled(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

239

iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/amd/init.c

242

static void init_translation_status(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2436

struct amd_iommu *iommu = irqd->chip_data;

drivers/iommu/amd/init.c

2446

writeq(xt.capxt, iommu->mmio_base + irqd->hwirq);

drivers/iommu/amd/init.c

2451

struct amd_iommu *iommu = irqd->chip_data;

drivers/iommu/amd/init.c

2453

writeq(0, iommu->mmio_base + irqd->hwirq);

drivers/iommu/amd/init.c

246

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

248

iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/amd/init.c

2516

static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname,

drivers/iommu/amd/init.c

2522

int node = dev_to_node(&iommu->dev->dev);

drivers/iommu/amd/init.c

2530

info.data = iommu;

drivers/iommu/amd/init.c

2540

iommu);

drivers/iommu/amd/init.c

2550

static int iommu_setup_intcapxt(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2554

snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name),

drivers/iommu/amd/init.c

2555

"AMD-Vi%d-Evt", iommu->index);

drivers/iommu/amd/init.c

2556

ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name,

drivers/iommu/amd/init.c

2562

snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name),

drivers/iommu/amd/init.c

2563

"AMD-Vi%d-PPR", iommu->index);

drivers/iommu/amd/init.c

2564

ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name,

drivers/iommu/amd/init.c

2571

snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name),

drivers/iommu/amd/init.c

2572

"AMD-Vi%d-GA", iommu->index);

drivers/iommu/amd/init.c

2573

ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name,

drivers/iommu/amd/init.c

2581

static int iommu_init_irq(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2585

if (iommu->int_enabled)

drivers/iommu/amd/init.c

2589

ret = iommu_setup_intcapxt(iommu);

drivers/iommu/amd/init.c

2590

else if (iommu->dev->msi_cap)

drivers/iommu/amd/init.c

2591

ret = iommu_setup_msi(iommu);

drivers/iommu/amd/init.c

2598

iommu->int_enabled = true;

drivers/iommu/amd/init.c

2602

iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);

drivers/iommu/amd/init.c

2604

iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);

drivers/iommu/amd/init.c

267

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

269

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

270

u64 tmp = iommu->features;

drivers/iommu/amd/init.c

271

u64 tmp2 = iommu->features2;

drivers/iommu/amd/init.c

273

if (list_is_first(&iommu->list, &amd_iommu_list)) {

drivers/iommu/amd/init.c

2759

static void iommu_init_flags(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2761

iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?

drivers/iommu/amd/init.c

2762

iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :

drivers/iommu/amd/init.c

2763

iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);

drivers/iommu/amd/init.c

2765

iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?

drivers/iommu/amd/init.c

2766

iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :

drivers/iommu/amd/init.c

2767

iommu_feature_disable(iommu, CONTROL_PASSPW_EN);

drivers/iommu/amd/init.c

2769

iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?

drivers/iommu/amd/init.c

2770

iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :

drivers/iommu/amd/init.c

2771

iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);

drivers/iommu/amd/init.c

2773

iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?

drivers/iommu/amd/init.c

2774

iommu_feature_enable(iommu, CONTROL_ISOC_EN) :

drivers/iommu/amd/init.c

2775

iommu_feature_disable(iommu, CONTROL_ISOC_EN);

drivers/iommu/amd/init.c

2780

iommu_feature_enable(iommu, CONTROL_COHERENT_EN);

drivers/iommu/amd/init.c

2783

iommu_feature_set(iommu, CTRL_INV_TO_1S, CTRL_INV_TO_MASK, CONTROL_INV_TIMEOUT);

drivers/iommu/amd/init.c

2787

iommu_feature_enable(iommu, CONTROL_EPH_EN);

drivers/iommu/amd/init.c

2790

static void iommu_apply_resume_quirks(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2794

struct pci_dev *pdev = iommu->root_pdev;

drivers/iommu/amd/init.c

2797

if (!is_rd890_iommu(iommu->dev) || !pdev)

drivers/iommu/amd/init.c

2814

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

drivers/iommu/amd/init.c

2815

iommu->stored_addr_lo);

drivers/iommu/amd/init.c

2816

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,

drivers/iommu/amd/init.c

2817

iommu->stored_addr_hi);

drivers/iommu/amd/init.c

2822

iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);

drivers/iommu/amd/init.c

2826

iommu_write_l2(iommu, i, iommu->stored_l2[i]);

drivers/iommu/amd/init.c

2829

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

drivers/iommu/amd/init.c

2830

iommu->stored_addr_lo | 1);

drivers/iommu/amd/init.c

2833

static void iommu_enable_ga(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2839

iommu_feature_enable(iommu, CONTROL_GA_EN);

drivers/iommu/amd/init.c

2840

iommu->irte_ops = &irte_128_ops;

drivers/iommu/amd/init.c

2843

iommu->irte_ops = &irte_32_ops;

drivers/iommu/amd/init.c

2849

static void iommu_disable_irtcachedis(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2851

iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);

drivers/iommu/amd/init.c

2854

static void iommu_enable_irtcachedis(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

286

iommu->index, iommu->pci_seg->id,

drivers/iommu/amd/init.c

2866

iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);

drivers/iommu/amd/init.c

2867

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

287

PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),

drivers/iommu/amd/init.c

2870

iommu->irtcachedis_enabled = true;

drivers/iommu/amd/init.c

2872

iommu->index, iommu->devid,

drivers/iommu/amd/init.c

2873

iommu->irtcachedis_enabled ? "disabled" : "enabled");

drivers/iommu/amd/init.c

2876

static void iommu_enable_2k_int(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

288

PCI_FUNC(iommu->devid));

drivers/iommu/amd/init.c

2881

iommu_feature_set(iommu,

drivers/iommu/amd/init.c

2887

static void early_enable_iommu(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2889

iommu_disable(iommu);

drivers/iommu/amd/init.c

2890

iommu_init_flags(iommu);

drivers/iommu/amd/init.c

2891

iommu_set_device_table(iommu);

drivers/iommu/amd/init.c

2892

iommu_enable_command_buffer(iommu);

drivers/iommu/amd/init.c

2893

iommu_enable_event_buffer(iommu);

drivers/iommu/amd/init.c

2894

iommu_set_exclusion_range(iommu);

drivers/iommu/amd/init.c

2895

iommu_enable_gt(iommu);

drivers/iommu/amd/init.c

2896

iommu_enable_ga(iommu);

drivers/iommu/amd/init.c

2897

iommu_enable_xt(iommu);

drivers/iommu/amd/init.c

2898

iommu_enable_irtcachedis(iommu);

drivers/iommu/amd/init.c

2899

iommu_enable_2k_int(iommu);

drivers/iommu/amd/init.c

2900

iommu_enable(iommu);

drivers/iommu/amd/init.c

2901

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

2914

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2940

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2941

clear_translation_pre_enabled(iommu);

drivers/iommu/amd/init.c

2942

early_enable_iommu(iommu);

drivers/iommu/amd/init.c

2952

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2953

iommu_disable_command_buffer(iommu);

drivers/iommu/amd/init.c

2954

iommu_disable_event_buffer(iommu);

drivers/iommu/amd/init.c

2955

iommu_disable_irtcachedis(iommu);

drivers/iommu/amd/init.c

2956

iommu_enable_command_buffer(iommu);

drivers/iommu/amd/init.c

2957

iommu_enable_event_buffer(iommu);

drivers/iommu/amd/init.c

2958

iommu_enable_ga(iommu);

drivers/iommu/amd/init.c

2959

iommu_enable_xt(iommu);

drivers/iommu/amd/init.c

2960

iommu_enable_irtcachedis(iommu);

drivers/iommu/amd/init.c

2961

iommu_enable_2k_int(iommu);

drivers/iommu/amd/init.c

2962

iommu_set_device_table(iommu);

drivers/iommu/amd/init.c

2963

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

2970

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2975

for_each_iommu(iommu)

drivers/iommu/amd/init.c

2976

amd_iommu_enable_ppr_log(iommu);

drivers/iommu/amd/init.c

2983

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2985

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2990

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

2994

iommu_feature_disable(iommu, CONTROL_GALOG_EN);

drivers/iommu/amd/init.c

2995

iommu_feature_disable(iommu, CONTROL_GAINT_EN);

drivers/iommu/amd/init.c

3002

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

302

static void __init early_iommu_features_init(struct amd_iommu *iommu,

drivers/iommu/amd/init.c

3026

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

3027

if (iommu_init_ga_log(iommu) ||

drivers/iommu/amd/init.c

3028

iommu_ga_log_enable(iommu))

drivers/iommu/amd/init.c

3031

iommu_feature_enable(iommu, CONTROL_GAM_EN);

drivers/iommu/amd/init.c

3033

iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);

drivers/iommu/amd/init.c

3043

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3045

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3046

iommu_disable(iommu);

drivers/iommu/amd/init.c

306

iommu->features = h->efr_reg;

drivers/iommu/amd/init.c

3061

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3063

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3064

iommu_apply_resume_quirks(iommu);

drivers/iommu/amd/init.c

3067

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3068

early_enable_iommu(iommu);

drivers/iommu/amd/init.c

307

iommu->features2 = h->efr_reg2;

drivers/iommu/amd/init.c

315

static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)

drivers/iommu/amd/init.c

319

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

drivers/iommu/amd/init.c

320

pci_read_config_dword(iommu->dev, 0xfc, &val);

drivers/iommu/amd/init.c

324

static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)

drivers/iommu/amd/init.c

326

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));

drivers/iommu/amd/init.c

327

pci_write_config_dword(iommu->dev, 0xfc, val);

drivers/iommu/amd/init.c

328

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

drivers/iommu/amd/init.c

3300

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3303

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

3304

ret = iommu_init_irq(iommu);

drivers/iommu/amd/init.c

331

static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)

drivers/iommu/amd/init.c

335

pci_write_config_dword(iommu->dev, 0xf0, address);

drivers/iommu/amd/init.c

336

pci_read_config_dword(iommu->dev, 0xf4, &val);

drivers/iommu/amd/init.c

340

static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)

drivers/iommu/amd/init.c

342

pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));

drivers/iommu/amd/init.c

343

pci_write_config_dword(iommu->dev, 0xf4, val);

drivers/iommu/amd/init.c

3465

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3471

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3472

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

359

static void iommu_set_exclusion_range(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

361

u64 start = iommu->exclusion_start & PAGE_MASK;

drivers/iommu/amd/init.c

362

u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;

drivers/iommu/amd/init.c

3621

x86_init.iommu.iommu_init = amd_iommu_init;

drivers/iommu/amd/init.c

365

if (!iommu->exclusion_start)

drivers/iommu/amd/init.c

369

memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,

drivers/iommu/amd/init.c

373

memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,

drivers/iommu/amd/init.c

377

static void iommu_set_cwwb_range(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

379

u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

3878

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3880

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3882

return iommu;

drivers/iommu/amd/init.c

389

memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,

drivers/iommu/amd/init.c

3895

struct amd_iommu *iommu = get_amd_iommu(idx);

drivers/iommu/amd/init.c

3897

if (iommu)

drivers/iommu/amd/init.c

3898

return iommu->max_banks;

drivers/iommu/amd/init.c

3910

struct amd_iommu *iommu = get_amd_iommu(idx);

drivers/iommu/amd/init.c

3912

if (iommu)

drivers/iommu/amd/init.c

3913

return iommu->max_counters;

drivers/iommu/amd/init.c

3918

static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,

drivers/iommu/amd/init.c

3929

if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))

drivers/iommu/amd/init.c

3935

max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |

drivers/iommu/amd/init.c

3936

(iommu->max_counters << 8) | 0x28);

drivers/iommu/amd/init.c

3944

writel((u32)val, iommu->mmio_base + offset);

drivers/iommu/amd/init.c

3945

writel((val >> 32), iommu->mmio_base + offset + 4);

drivers/iommu/amd/init.c

3947

*value = readl(iommu->mmio_base + offset + 4);

drivers/iommu/amd/init.c

3949

*value |= readl(iommu->mmio_base + offset);

drivers/iommu/amd/init.c

3956

int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)

drivers/iommu/amd/init.c

3958

if (!iommu)

drivers/iommu/amd/init.c

396

memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,

drivers/iommu/amd/init.c

3961

return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);

drivers/iommu/amd/init.c

3964

int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)

drivers/iommu/amd/init.c

3966

if (!iommu)

drivers/iommu/amd/init.c

3969

return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);

drivers/iommu/amd/init.c

401

static void iommu_set_device_table(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

4030

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

4036

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

4037

ret = iommu_make_shared(iommu->evt_buf, EVT_BUFFER_SIZE);

drivers/iommu/amd/init.c

404

u32 dev_table_size = iommu->pci_seg->dev_table_size;

drivers/iommu/amd/init.c

4041

ret = iommu_make_shared(iommu->ppr_log, PPR_LOG_SIZE);

drivers/iommu/amd/init.c

4045

ret = iommu_make_shared((void *)iommu->cmd_sem, PAGE_SIZE);

drivers/iommu/amd/init.c

405

void *dev_table = (void *)get_dev_table(iommu);

drivers/iommu/amd/init.c

407

BUG_ON(iommu->mmio_base == NULL);

drivers/iommu/amd/init.c

414

memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,

drivers/iommu/amd/init.c

418

static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift)

drivers/iommu/amd/init.c

422

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

426

writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

430

void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)

drivers/iommu/amd/init.c

432

iommu_feature_set(iommu, 1ULL, 1ULL, bit);

drivers/iommu/amd/init.c

435

static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)

drivers/iommu/amd/init.c

437

iommu_feature_set(iommu, 0ULL, 1ULL, bit);

drivers/iommu/amd/init.c

441

static void iommu_enable(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

443

iommu_feature_enable(iommu, CONTROL_IOMMU_EN);

drivers/iommu/amd/init.c

446

static void iommu_disable(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

448

if (!iommu->mmio_base)

drivers/iommu/amd/init.c

452

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

455

iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);

drivers/iommu/amd/init.c

456

iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);

drivers/iommu/amd/init.c

459

iommu_feature_disable(iommu, CONTROL_GALOG_EN);

drivers/iommu/amd/init.c

460

iommu_feature_disable(iommu, CONTROL_GAINT_EN);

drivers/iommu/amd/init.c

463

iommu_feature_disable(iommu, CONTROL_PPRLOG_EN);

drivers/iommu/amd/init.c

464

iommu_feature_disable(iommu, CONTROL_PPRINT_EN);

drivers/iommu/amd/init.c

467

iommu_feature_disable(iommu, CONTROL_IOMMU_EN);

drivers/iommu/amd/init.c

470

iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);

drivers/iommu/amd/init.c

489

static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

491

if (iommu->mmio_base)

drivers/iommu/amd/init.c

492

iounmap(iommu->mmio_base);

drivers/iommu/amd/init.c

493

release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);

drivers/iommu/amd/init.c

741

static int __init alloc_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

743

iommu->cmd_buf = iommu_alloc_pages_sz(GFP_KERNEL, CMD_BUFFER_SIZE);

drivers/iommu/amd/init.c

745

return iommu->cmd_buf ? 0 : -ENOMEM;

drivers/iommu/amd/init.c

752

void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,

drivers/iommu/amd/init.c

758

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

764

iommu_feature_disable(iommu, cntrl_log);

drivers/iommu/amd/init.c

765

iommu_feature_disable(iommu, cntrl_intr);

drivers/iommu/amd/init.c

767

writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

769

iommu_feature_enable(iommu, cntrl_intr);

drivers/iommu/amd/init.c

770

iommu_feature_enable(iommu, cntrl_log);

drivers/iommu/amd/init.c

777

void amd_iommu_restart_event_logging(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

779

amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN,

drivers/iommu/amd/init.c

788

void amd_iommu_restart_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

790

amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN,

drivers/iommu/amd/init.c

799

static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

801

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

803

writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

drivers/iommu/amd/init.c

804

writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/init.c

805

iommu->cmd_buf_head = 0;

drivers/iommu/amd/init.c

806

iommu->cmd_buf_tail = 0;

drivers/iommu/amd/init.c

808

iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

815

static void iommu_enable_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

819

BUG_ON(iommu->cmd_buf == NULL);

drivers/iommu/amd/init.c

826

entry = iommu_virt_to_phys(iommu->cmd_buf);

drivers/iommu/amd/init.c

828

memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,

drivers/iommu/amd/init.c

832

amd_iommu_reset_cmd_buffer(iommu);

drivers/iommu/amd/init.c

838

static void iommu_disable_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

840

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

843

static void __init free_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

845

iommu_free_pages(iommu->cmd_buf);

drivers/iommu/amd/init.c

848

void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,

drivers/iommu/amd/init.c

867

static int __init alloc_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

869

iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,

drivers/iommu/amd/init.c

872

return iommu->evt_buf ? 0 : -ENOMEM;

drivers/iommu/amd/init.c

875

static void iommu_enable_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

879

BUG_ON(iommu->evt_buf == NULL);

drivers/iommu/amd/init.c

886

entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;

drivers/iommu/amd/init.c

887

memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,

drivers/iommu/amd/init.c

892

writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

drivers/iommu/amd/init.c

893

writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

drivers/iommu/amd/init.c

895

iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);

drivers/iommu/amd/init.c

901

static void iommu_disable_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

903

iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);

drivers/iommu/amd/init.c

906

static void __init free_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

908

iommu_free_pages(iommu->evt_buf);

drivers/iommu/amd/init.c

911

static void free_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

914

iommu_free_pages(iommu->ga_log);

drivers/iommu/amd/init.c

915

iommu_free_pages(iommu->ga_log_tail);

drivers/iommu/amd/init.c

920

static int iommu_ga_log_enable(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

925

if (!iommu->ga_log)

drivers/iommu/amd/init.c

928

entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;

drivers/iommu/amd/init.c

929

memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,

drivers/iommu/amd/init.c

931

entry = (iommu_virt_to_phys(iommu->ga_log_tail) &

drivers/iommu/amd/init.c

933

memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,

drivers/iommu/amd/init.c

935

writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

drivers/iommu/amd/init.c

936

writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);

drivers/iommu/amd/init.c

939

iommu_feature_enable(iommu, CONTROL_GAINT_EN);

drivers/iommu/amd/init.c

940

iommu_feature_enable(iommu, CONTROL_GALOG_EN);

drivers/iommu/amd/init.c

943

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

955

static int iommu_init_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

960

iommu->ga_log = iommu_alloc_pages_sz(GFP_KERNEL, GA_LOG_SIZE);

drivers/iommu/amd/init.c

961

if (!iommu->ga_log)

drivers/iommu/amd/init.c

964

iommu->ga_log_tail = iommu_alloc_pages_sz(GFP_KERNEL, 8);

drivers/iommu/amd/init.c

965

if (!iommu->ga_log_tail)

drivers/iommu/amd/init.c

970

free_ga_log(iommu);

drivers/iommu/amd/init.c

975

static int __init alloc_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

977

iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1);

drivers/iommu/amd/init.c

978

if (!iommu->cmd_sem)

drivers/iommu/amd/init.c

980

iommu->cmd_sem_paddr = iommu_virt_to_phys((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

984

static int __init remap_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

989

paddr = readq(iommu->mmio_base + MMIO_EVT_BUF_OFFSET) & PM_ADDR_MASK;

drivers/iommu/amd/init.c

990

iommu->evt_buf = iommu_memremap(paddr, EVT_BUFFER_SIZE);

drivers/iommu/amd/init.c

992

return iommu->evt_buf ? 0 : -ENOMEM;

drivers/iommu/amd/init.c

995

static int __init remap_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1001

static void iommu_poll_events(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1005

head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1006

tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1009

iommu_print_event(iommu, iommu->evt_buf + head);

drivers/iommu/amd/iommu.c

1013

writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1037

static void iommu_poll_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1041

if (iommu->ga_log == NULL)

drivers/iommu/amd/iommu.c

1044

head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1045

tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1051

raw = (u64 *)(iommu->ga_log + head);

drivers/iommu/amd/iommu.c

1058

writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1080

amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1086

dev_set_msi_domain(dev, iommu->ir_domain);

drivers/iommu/amd/iommu.c

1091

amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }

drivers/iommu/amd/iommu.c

1099

struct amd_iommu *iommu = (struct amd_iommu *) data;

drivers/iommu/amd/iommu.c

1100

u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/iommu.c

1105

writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/iommu.c

1109

iommu->index, evt_type);

drivers/iommu/amd/iommu.c

1110

int_handler(iommu);

drivers/iommu/amd/iommu.c

1114

overflow_handler(iommu);

drivers/iommu/amd/iommu.c

1129

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/iommu.c

1177

static void dump_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1183

head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1184

tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1190

cmd = (struct iommu_cmd *)(iommu->cmd_buf + i * sizeof(*cmd));

drivers/iommu/amd/iommu.c

1196

static int wait_on_sem(struct amd_iommu *iommu, u64 data)

drivers/iommu/amd/iommu.c

1204

while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 &&

drivers/iommu/amd/iommu.c

1213

iommu->pci_seg->id, PCI_BUS_NUM(iommu->devid),

drivers/iommu/amd/iommu.c

1214

PCI_SLOT(iommu->devid), PCI_FUNC(iommu->devid));

drivers/iommu/amd/iommu.c

1217

DO_ONCE_LITE(dump_command_buffer, iommu);

drivers/iommu/amd/iommu.c

1225

static void copy_cmd_to_buffer(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1232

tail = iommu->cmd_buf_tail;

drivers/iommu/amd/iommu.c

1233

target = iommu->cmd_buf + tail;

drivers/iommu/amd/iommu.c

1237

iommu->cmd_buf_tail = tail;

drivers/iommu/amd/iommu.c

1240

writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1244

struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1247

u64 paddr = iommu->cmd_sem_paddr;

drivers/iommu/amd/iommu.c

1381

static int __iommu_queue_command_sync(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1388

next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;

drivers/iommu/amd/iommu.c

1390

left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;

drivers/iommu/amd/iommu.c

1404

iommu->cmd_buf_head = readl(iommu->mmio_base +

drivers/iommu/amd/iommu.c

1410

copy_cmd_to_buffer(iommu, cmd);

drivers/iommu/amd/iommu.c

1413

iommu->need_sync = sync;

drivers/iommu/amd/iommu.c

1418

static int iommu_queue_command_sync(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1425

raw_spin_lock_irqsave(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1426

ret = __iommu_queue_command_sync(iommu, cmd, sync);

drivers/iommu/amd/iommu.c

1427

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1432

static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)

drivers/iommu/amd/iommu.c

1434

return iommu_queue_command_sync(iommu, cmd, true);

drivers/iommu/amd/iommu.c

1437

static u64 get_cmdsem_val(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1439

lockdep_assert_held(&iommu->lock);

drivers/iommu/amd/iommu.c

1440

return ++iommu->cmd_sem_val;

drivers/iommu/amd/iommu.c

1447

static int iommu_completion_wait(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1454

if (!iommu->need_sync)

drivers/iommu/amd/iommu.c

1457

raw_spin_lock_irqsave(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1459

data = get_cmdsem_val(iommu);

drivers/iommu/amd/iommu.c

146

static void update_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data,

drivers/iommu/amd/iommu.c

1460

build_completion_wait(&cmd, iommu, data);

drivers/iommu/amd/iommu.c

1462

ret = __iommu_queue_command_sync(iommu, &cmd, false);

drivers/iommu/amd/iommu.c

1463

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1468

ret = wait_on_sem(iommu, data);

drivers/iommu/amd/iommu.c

1485

iommu_completion_wait(pdom_iommu_info->iommu);

drivers/iommu/amd/iommu.c

1488

static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

1494

return iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1497

static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

150

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/iommu.c

1501

ret = iommu_flush_dte(iommu, devid);

drivers/iommu/amd/iommu.c

1503

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1506

static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1509

u16 last_bdf = iommu->pci_seg->last_bdf;

drivers/iommu/amd/iommu.c

1512

iommu_flush_dte(iommu, devid);

drivers/iommu/amd/iommu.c

1514

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1521

static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1524

u16 last_bdf = iommu->pci_seg->last_bdf;

drivers/iommu/amd/iommu.c

1530

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1533

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1536

static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)

drivers/iommu/amd/iommu.c

1542

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1544

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1555

struct amd_iommu *iommu = container_of(aviommu->core.iommu_dev,

drivers/iommu/amd/iommu.c

1556

struct amd_iommu, iommu);

drivers/iommu/amd/iommu.c

1563

iommu->devid, gdom_info->hdom_id);

drivers/iommu/amd/iommu.c

1566

ret |= iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1573

static void amd_iommu_flush_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1579

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1580

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1583

static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

1589

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

159

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1592

static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1595

u16 last_bdf = iommu->pci_seg->last_bdf;

drivers/iommu/amd/iommu.c

1597

if (iommu->irtcachedis_enabled)

drivers/iommu/amd/iommu.c

1601

iommu_flush_irt(iommu, devid);

drivers/iommu/amd/iommu.c

1603

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1606

void amd_iommu_flush_all_caches(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1609

amd_iommu_flush_all(iommu);

drivers/iommu/amd/iommu.c

1611

amd_iommu_flush_dte_all(iommu);

drivers/iommu/amd/iommu.c

1612

amd_iommu_flush_irt_all(iommu);

drivers/iommu/amd/iommu.c

1613

amd_iommu_flush_tlb_all(iommu);

drivers/iommu/amd/iommu.c

1623

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

1630

return iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1635

struct amd_iommu *iommu = data;

drivers/iommu/amd/iommu.c

1637

return iommu_flush_dte(iommu, alias);

drivers/iommu/amd/iommu.c

164

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1645

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

1656

device_flush_dte_alias, iommu);

drivers/iommu/amd/iommu.c

1658

ret = iommu_flush_dte(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1662

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

1665

ret = iommu_flush_dte(iommu, alias);

drivers/iommu/amd/iommu.c

1688

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/iommu.c

1694

ret |= iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1718

ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd);

drivers/iommu/amd/iommu.c

172

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

181

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1830

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/iommu.c

1834

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1839

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1852

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

1856

iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

1861

return iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

194

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1958

struct amd_iommu *iommu, int pasids)

drivers/iommu/amd/iommu.c

1961

int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;

drivers/iommu/amd/iommu.c

199

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

212

void amd_iommu_update_dte(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

2154

static void set_dte_entry(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

216

update_dte256(iommu, dev_data, new);

drivers/iommu/amd/iommu.c

2162

struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];

drivers/iommu/amd/iommu.c

217

clone_aliases(iommu, dev_data->dev);

drivers/iommu/amd/iommu.c

2177

amd_iommu_update_dte(iommu, dev_data, &new);

drivers/iommu/amd/iommu.c

2185

amd_iommu_flush_tlb_domid(iommu, old_domid);

drivers/iommu/amd/iommu.c

219

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

2192

static void clear_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data)

drivers/iommu/amd/iommu.c

2197

amd_iommu_update_dte(iommu, dev_data, &new);

drivers/iommu/amd/iommu.c

2203

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/iommu.c

2206

set_dte_entry(iommu, dev_data, 0, 0);

drivers/iommu/amd/iommu.c

2208

clear_dte_entry(iommu, dev_data);

drivers/iommu/amd/iommu.c

2218

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

222

static void get_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data,

drivers/iommu/amd/iommu.c

2234

ret = setup_gcr3_table(&dev_data->gcr3_info, iommu,

drivers/iommu/amd/iommu.c

2265

static int pdom_attach_iommu(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

227

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/iommu.c

2274

pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);

drivers/iommu/amd/iommu.c

2286

pdom_iommu_info->iommu = iommu;

drivers/iommu/amd/iommu.c

2289

curr = xa_cmpxchg(&pdom->iommu_array, iommu->index,

drivers/iommu/amd/iommu.c

2302

static void pdom_detach_iommu(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

2310

pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);

drivers/iommu/amd/iommu.c

2318

xa_erase(&pdom->iommu_array, iommu->index);

drivers/iommu/amd/iommu.c

2333

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

2346

ret = pdom_attach_iommu(iommu, domain);

drivers/iommu/amd/iommu.c

2354

pdom_detach_iommu(iommu, domain);

drivers/iommu/amd/iommu.c

2368

if (amd_iommu_iopf_add_device(iommu, dev_data))

drivers/iommu/amd/iommu.c

2395

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

2413

amd_iommu_iopf_remove_device(iommu, dev_data);

drivers/iommu/amd/iommu.c

2436

pdom_detach_iommu(iommu, domain);

drivers/iommu/amd/iommu.c

2445

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

2452

iommu = rlookup_amd_iommu(dev);

drivers/iommu/amd/iommu.c

2453

if (!iommu)

drivers/iommu/amd/iommu.c

2457

if (!iommu->iommu.ops)

drivers/iommu/amd/iommu.c

2461

return &iommu->iommu;

drivers/iommu/amd/iommu.c

2463

ret = iommu_init_device(iommu, dev);

drivers/iommu/amd/iommu.c

2467

iommu_ignore_device(iommu, dev);

drivers/iommu/amd/iommu.c

2471

amd_iommu_set_pci_msi_domain(dev, iommu);

drivers/iommu/amd/iommu.c

2472

iommu_dev = &iommu->iommu;

drivers/iommu/amd/iommu.c

2481

dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,

drivers/iommu/amd/iommu.c

2492

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

2566

static bool amd_iommu_hd_support(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

2571

return iommu && (iommu->features & FEATURE_HDSUP);

drivers/iommu/amd/iommu.c

2577

container_of(iommupt, struct protection_domain, iommu);

drivers/iommu/amd/iommu.c

2589

container_of(iommu_table, struct protection_domain, iommu);

drivers/iommu/amd/iommu.c

2596

struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev);

drivers/iommu/amd/iommu.c

2599

set_dte_entry(iommu, dev_data, top_paddr, top_level);

drivers/iommu/amd/iommu.c

2600

clone_aliases(iommu, dev_data->dev);

drivers/iommu/amd/iommu.c

2687

domain->iommu.driver_ops = &amd_hw_driver_ops_v1;

drivers/iommu/amd/iommu.c

2688

domain->iommu.nid = dev_to_node(dev);

drivers/iommu/amd/iommu.c

2769

domain->iommu.nid = dev_to_node(dev);

drivers/iommu/amd/iommu.c

2816

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

2833

!amd_iommu_hd_support(iommu))

drivers/iommu/amd/iommu.c

2872

pt_iommu_deinit(&domain->iommu);

drivers/iommu/amd/iommu.c

2947

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

2963

if (dom->dirty_ops && !amd_iommu_hd_support(iommu))

drivers/iommu/amd/iommu.c

2997

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

2999

return amd_iommu_hd_support(iommu);

drivers/iommu/amd/iommu.c

3015

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

3027

iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

3028

dte = &get_dev_table(iommu)[dev_data->devid];

drivers/iommu/amd/iommu.c

3054

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

3063

iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

3064

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

312

struct dev_table_entry *get_dev_table(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

315

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3182

static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3189

if (iommu->irtcachedis_enabled)

drivers/iommu/amd/iommu.c

3194

raw_spin_lock_irqsave(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

3195

data = get_cmdsem_val(iommu);

drivers/iommu/amd/iommu.c

3196

build_completion_wait(&cmd2, iommu, data);

drivers/iommu/amd/iommu.c

3198

ret = __iommu_queue_command_sync(iommu, &cmd, true);

drivers/iommu/amd/iommu.c

3201

ret = __iommu_queue_command_sync(iommu, &cmd2, false);

drivers/iommu/amd/iommu.c

3204

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

3206

wait_on_sem(iommu, data);

drivers/iommu/amd/iommu.c

3210

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

3220

static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,

drivers/iommu/amd/iommu.c

3224

struct dev_table_entry *dte = &get_dev_table(iommu)[devid];

drivers/iommu/amd/iommu.c

3225

struct iommu_dev_data *dev_data = search_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

3242

static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3245

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3279

static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,

drivers/iommu/amd/iommu.c

3282

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3285

set_dte_irq_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3286

iommu_flush_dte(iommu, devid);

drivers/iommu/amd/iommu.c

3294

struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);

drivers/iommu/amd/iommu.c

3296

if (!iommu)

drivers/iommu/amd/iommu.c

3299

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3301

set_dte_irq_entry(iommu, alias, table);

drivers/iommu/amd/iommu.c

3315

static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

3323

int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;

drivers/iommu/amd/iommu.c

3328

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3336

set_remap_table_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3354

set_remap_table_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3365

set_remap_table_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3368

set_remap_table_entry(iommu, alias, table);

drivers/iommu/amd/iommu.c

3371

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

3383

static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,

drivers/iommu/amd/iommu.c

3391

table = alloc_irq_table(iommu, devid, pdev, max_irqs);

drivers/iommu/amd/iommu.c

3403

if (!iommu->irte_ops->is_allocated(table, index)) {

drivers/iommu/amd/iommu.c

3413

iommu->irte_ops->set_allocated(table, index - c + 1);

drivers/iommu/amd/iommu.c

342

void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3430

static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,

drivers/iommu/amd/iommu.c

3438

table = get_irq_table(iommu, devid);

drivers/iommu/amd/iommu.c

344

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

346

pci_seg->rlookup_table[devid] = iommu;

drivers/iommu/amd/iommu.c

3461

static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,

drivers/iommu/amd/iommu.c

3466

ret = __modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3470

iommu_flush_irt_and_complete(iommu, devid);

drivers/iommu/amd/iommu.c

3475

static int modify_irte(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

3481

table = get_irq_table(iommu, devid);

drivers/iommu/amd/iommu.c

3489

iommu_flush_irt_and_complete(iommu, devid);

drivers/iommu/amd/iommu.c

3494

static void free_irte(struct amd_iommu *iommu, u16 devid, int index)

drivers/iommu/amd/iommu.c

3499

table = get_irq_table(iommu, devid);

drivers/iommu/amd/iommu.c

3504

iommu->irte_ops->clear_allocated(table, index);

drivers/iommu/amd/iommu.c

3507

iommu_flush_irt_and_complete(iommu, devid);

drivers/iommu/amd/iommu.c

3540

static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3545

modify_irte(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3548

static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3553

modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3556

static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3561

modify_irte(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3564

static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3569

modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3572

static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,

drivers/iommu/amd/iommu.c

3579

modify_irte(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3582

static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,

drivers/iommu/amd/iommu.c

3593

modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3689

struct amd_iommu *iommu = data->iommu;

drivers/iommu/amd/iommu.c

3691

if (!iommu)

drivers/iommu/amd/iommu.c

3696

iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED,

drivers/iommu/amd/iommu.c

370

static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

373

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3740

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

3758

iommu = __rlookup_amd_iommu(seg, devid);

drivers/iommu/amd/iommu.c

3759

if (!iommu)

drivers/iommu/amd/iommu.c

3762

dev_data = search_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

3772

table = alloc_irq_table(iommu, devid, NULL, max_irqs);

drivers/iommu/amd/iommu.c

3781

iommu->irte_ops->set_allocated(table, i);

drivers/iommu/amd/iommu.c

3792

index = alloc_irq_index(iommu, devid, nr_irqs, align,

drivers/iommu/amd/iommu.c

3796

index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL,

drivers/iommu/amd/iommu.c

3828

data->iommu = iommu;

drivers/iommu/amd/iommu.c

3844

free_irte(iommu, devid, index + i);

drivers/iommu/amd/iommu.c

3863

free_irte(data->iommu, irte_info->devid, irte_info->index);

drivers/iommu/amd/iommu.c

3871

static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

388

struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3881

struct amd_iommu *iommu = data->iommu;

drivers/iommu/amd/iommu.c

3884

if (!iommu)

drivers/iommu/amd/iommu.c

3887

iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,

drivers/iommu/amd/iommu.c

3889

amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);

drivers/iommu/amd/iommu.c

3898

struct amd_iommu *iommu = data->iommu;

drivers/iommu/amd/iommu.c

3900

if (iommu)

drivers/iommu/amd/iommu.c

3901

iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,

drivers/iommu/amd/iommu.c

3908

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

392

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3921

iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));

drivers/iommu/amd/iommu.c

3923

return iommu && iommu->ir_domain == d;

drivers/iommu/amd/iommu.c

3978

if (!ir_data->iommu)

drivers/iommu/amd/iommu.c

3983

return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

drivers/iommu/amd/iommu.c

4013

return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

drivers/iommu/amd/iommu.c

4045

return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

drivers/iommu/amd/iommu.c

4061

if (ir_data->iommu == NULL)

drivers/iommu/amd/iommu.c

4064

dev_data = search_dev_data(ir_data->iommu, irte_info->devid);

drivers/iommu/amd/iommu.c

409

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

4094

static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

4104

iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,

drivers/iommu/amd/iommu.c

4116

struct amd_iommu *iommu = ir_data->iommu;

drivers/iommu/amd/iommu.c

4119

if (!iommu)

drivers/iommu/amd/iommu.c

4126

amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg);

drivers/iommu/amd/iommu.c

4160

int amd_iommu_create_irq_domain(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

4163

.fwnode = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index),

drivers/iommu/amd/iommu.c

4166

.host_data = iommu,

drivers/iommu/amd/iommu.c

417

iommu = rlookup_amd_iommu(&pdev->dev);

drivers/iommu/amd/iommu.c

4173

iommu->ir_domain = msi_create_parent_irq_domain(&info, &amdvi_msi_parent_ops);

drivers/iommu/amd/iommu.c

4174

if (!iommu->ir_domain) {

drivers/iommu/amd/iommu.c

418

if (!iommu)

drivers/iommu/amd/iommu.c

428

get_dte256(iommu, dev_data, &new);

drivers/iommu/amd/iommu.c

431

alias_data = find_dev_data(iommu, alias);

drivers/iommu/amd/iommu.c

437

update_dte256(iommu, alias_data, &new);

drivers/iommu/amd/iommu.c

439

amd_iommu_set_rlookup_table(iommu, alias);

drivers/iommu/amd/iommu.c

444

static void clone_aliases(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

457

clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL);

drivers/iommu/amd/iommu.c

462

static void setup_aliases(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

465

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

481

clone_aliases(iommu, dev);

drivers/iommu/amd/iommu.c

484

static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

488

dev_data = search_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

491

dev_data = alloc_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

495

if (translation_pre_enabled(iommu))

drivers/iommu/amd/iommu.c

673

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

684

iommu = rlookup_amd_iommu(dev);

drivers/iommu/amd/iommu.c

685

if (!iommu)

drivers/iommu/amd/iommu.c

689

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

696

static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

709

dev_data = find_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

720

setup_aliases(iommu, dev);

drivers/iommu/amd/iommu.c

736

static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

738

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

739

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/iommu.c

75

static void set_dte_entry(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

750

setup_aliases(iommu, dev);

drivers/iommu/amd/iommu.c

760

static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

764

struct iommu_dev_data *dev_data = find_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

766

get_dte256(iommu, dev_data, &dte);

drivers/iommu/amd/iommu.c

781

static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event)

drivers/iommu/amd/iommu.c

793

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

drivers/iommu/amd/iommu.c

805

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

813

static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event)

drivers/iommu/amd/iommu.c

826

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

drivers/iommu/amd/iommu.c

838

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

84

static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/iommu.c

852

static void amd_iommu_report_page_fault(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

859

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

drivers/iommu/amd/iommu.c

86

static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/iommu.c

875

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),

drivers/iommu/amd/iommu.c

894

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

903

static void iommu_print_event(struct amd_iommu *iommu, void *__evt)

drivers/iommu/amd/iommu.c

905

struct device *dev = iommu->iommu.dev;

drivers/iommu/amd/iommu.c

91

static void clone_aliases(struct amd_iommu *iommu, struct device *dev);

drivers/iommu/amd/iommu.c

919

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/iommu.c

93

static int iommu_completion_wait(struct amd_iommu *iommu);

drivers/iommu/amd/iommu.c

932

amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);

drivers/iommu/amd/iommu.c

939

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

942

dump_dte_entry(iommu, devid);

drivers/iommu/amd/iommu.c

947

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

952

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

965

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

970

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

974

amd_iommu_report_rmp_fault(iommu, event);

drivers/iommu/amd/iommu.c

977

amd_iommu_report_rmp_hw_error(iommu, event);

drivers/iommu/amd/iommu.c

983

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/nested.c

186

static void set_dte_nested(struct amd_iommu *iommu, struct iommu_domain *dom,

drivers/iommu/amd/nested.c

237

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/nested.c

249

set_dte_nested(iommu, dom, dev_data, &new);

drivers/iommu/amd/nested.c

251

amd_iommu_update_dte(iommu, dev_data, &new);

drivers/iommu/amd/ppr.c

105

static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)

drivers/iommu/amd/ppr.c

117

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,

drivers/iommu/amd/ppr.c

122

if (!ppr_is_valid(iommu, raw))

drivers/iommu/amd/ppr.c

162

void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

166

if (iommu->ppr_log == NULL)

drivers/iommu/amd/ppr.c

169

head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

drivers/iommu/amd/ppr.c

170

tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

drivers/iommu/amd/ppr.c

177

raw = (u64 *)(iommu->ppr_log + head);

drivers/iommu/amd/ppr.c

20

int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

205

writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

drivers/iommu/amd/ppr.c

208

iommu_call_iopf_notifier(iommu, entry);

drivers/iommu/amd/ppr.c

218

int amd_iommu_iopf_init(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

22

iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,

drivers/iommu/amd/ppr.c

222

if (iommu->iopf_queue)

drivers/iommu/amd/ppr.c

225

snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), "amdvi-%#x",

drivers/iommu/amd/ppr.c

226

PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, iommu->devid));

drivers/iommu/amd/ppr.c

228

iommu->iopf_queue = iopf_queue_alloc(iommu->iopfq_name);

drivers/iommu/amd/ppr.c

229

if (!iommu->iopf_queue)

drivers/iommu/amd/ppr.c

236

void amd_iommu_iopf_uninit(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

238

iopf_queue_free(iommu->iopf_queue);

drivers/iommu/amd/ppr.c

239

iommu->iopf_queue = NULL;

drivers/iommu/amd/ppr.c

24

return iommu->ppr_log ? 0 : -ENOMEM;

drivers/iommu/amd/ppr.c

248

int amd_iommu_iopf_add_device(struct amd_iommu *iommu,

drivers/iommu/amd/ppr.c

256

if (!iommu->iopf_queue)

drivers/iommu/amd/ppr.c

259

ret = iopf_queue_add_device(iommu->iopf_queue, dev_data->dev);

drivers/iommu/amd/ppr.c

268

void amd_iommu_iopf_remove_device(struct amd_iommu *iommu,

drivers/iommu/amd/ppr.c

27

void amd_iommu_enable_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

271

iopf_queue_remove_device(iommu->iopf_queue, dev_data->dev);

drivers/iommu/amd/ppr.c

31

if (iommu->ppr_log == NULL)

drivers/iommu/amd/ppr.c

34

iommu_feature_enable(iommu, CONTROL_PPR_EN);

drivers/iommu/amd/ppr.c

36

entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;

drivers/iommu/amd/ppr.c

38

memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,

drivers/iommu/amd/ppr.c

42

writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

drivers/iommu/amd/ppr.c

43

writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

drivers/iommu/amd/ppr.c

45

iommu_feature_enable(iommu, CONTROL_PPRINT_EN);

drivers/iommu/amd/ppr.c

46

iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);

drivers/iommu/amd/ppr.c

49

void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

51

iommu_free_pages(iommu->ppr_log);

drivers/iommu/amd/ppr.c

58

void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

60

amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN,

drivers/iommu/amd/ppr.c

81

static bool ppr_is_valid(struct amd_iommu *iommu, u64 *raw)

drivers/iommu/amd/ppr.c

83

struct device *dev = iommu->iommu.dev;

drivers/iommu/amd/ppr.c

89

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/ppr.c

97

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/apple-dart.c

1184

ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",

drivers/iommu/apple-dart.c

1189

ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);

drivers/iommu/apple-dart.c

1201

iommu_device_sysfs_remove(&dart->iommu);

drivers/iommu/apple-dart.c

1217

iommu_device_unregister(&dart->iommu);

drivers/iommu/apple-dart.c

1218

iommu_device_sysfs_remove(&dart->iommu);

drivers/iommu/apple-dart.c

227

struct iommu_device iommu;

drivers/iommu/apple-dart.c

757

return &cfg->stream_maps[0].dart->iommu;

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c

451

container_of(viommu->iommu_dev, struct arm_smmu_device, iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

3638

return &smmu->iommu;

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

4500

smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

4913

ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

4918

ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

4927

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

4939

iommu_device_unregister(&smmu->iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

4940

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h

802

struct iommu_device iommu;

drivers/iommu/arm/arm-smmu/arm-smmu.c

1497

return &smmu->iommu;

drivers/iommu/arm/arm-smmu/arm-smmu.c

2235

err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,

drivers/iommu/arm/arm-smmu/arm-smmu.c

2240

err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,

drivers/iommu/arm/arm-smmu/arm-smmu.c

2243

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu/arm-smmu.c

2286

iommu_device_unregister(&smmu->iommu);

drivers/iommu/arm/arm-smmu/arm-smmu.c

2287

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu/arm-smmu.h

342

struct iommu_device iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

228

if (qcom_domain->iommu)

drivers/iommu/arm/arm-smmu/qcom_iommu.c

239

qcom_domain->iommu = qcom_iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

317

qcom_domain->iommu = NULL;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

347

if (qcom_domain->iommu) {

drivers/iommu/arm/arm-smmu/qcom_iommu.c

354

pm_runtime_get_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

356

pm_runtime_put_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

385

if (qcom_domain->iommu != qcom_iommu)

drivers/iommu/arm/arm-smmu/qcom_iommu.c

404

if (WARN_ON(!qcom_domain->iommu))

drivers/iommu/arm/arm-smmu/qcom_iommu.c

464

pm_runtime_get_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

468

pm_runtime_put_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

48

struct iommu_device iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

481

pm_runtime_get_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

483

pm_runtime_put_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

546

return &qcom_iommu->iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

71

struct qcom_iommu_dev *iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

84

struct qcom_iommu_dev *qcom_iommu = d->iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

848

ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,

drivers/iommu/arm/arm-smmu/qcom_iommu.c

855

ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

880

iommu_device_sysfs_remove(&qcom_iommu->iommu);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

881

iommu_device_unregister(&qcom_iommu->iommu);

drivers/iommu/dma-iommu.c

2116

dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac;

drivers/iommu/dma-iommu.c

640

if (dev->iommu->shadow_on_flush) {

drivers/iommu/dma-iommu.c

780

if (dma_limit > DMA_BIT_MASK(32) && dev->iommu->pci_32bit_workaround) {

drivers/iommu/dma-iommu.c

786

dev->iommu->pci_32bit_workaround = false;

drivers/iommu/exynos-iommu.c

1426

return &data->iommu;

drivers/iommu/exynos-iommu.c

304

struct iommu_device iommu; /* IOMMU core handle */

drivers/iommu/exynos-iommu.c

780

ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL,

drivers/iommu/exynos-iommu.c

816

ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev);

drivers/iommu/exynos-iommu.c

823

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/generic_pt/fmt/amdv1.h

331

return &container_of(iommu_table, struct pt_iommu_amdv1, iommu)

drivers/iommu/generic_pt/fmt/amdv1.h

337

return &container_of(common, struct pt_iommu_amdv1, amdpt.common)->iommu;

drivers/iommu/generic_pt/fmt/vtdss.h

205

return &container_of(iommu_table, struct pt_iommu_table, iommu)

drivers/iommu/generic_pt/fmt/vtdss.h

212

->iommu;

drivers/iommu/generic_pt/fmt/x86_64.h

206

return &container_of(iommu_table, struct pt_iommu_table, iommu)

drivers/iommu/generic_pt/fmt/x86_64.h

213

->iommu;

drivers/iommu/generic_pt/iommu_pt.h

1206

struct pt_iommu *iommu_table = &fmt_table->iommu;

drivers/iommu/generic_pt/iommu_pt.h

1209

static_assert(offsetof(struct pt_iommu_table, iommu.domain) == 0);

drivers/iommu/generic_pt/iommu_pt.h

1210

memset_after(fmt_table, 0, iommu.domain);

drivers/iommu/generic_pt/iommu_pt.h

1224

struct pt_iommu *iommu_table = &fmt_table->iommu;

drivers/iommu/generic_pt/iommu_pt.h

1285

struct pt_iommu *iommu_table = &fmt_table->iommu;

drivers/iommu/generic_pt/kunit_generic_pt.h

813

pt_iommu_deinit(priv->iommu);

drivers/iommu/generic_pt/kunit_iommu.h

140

priv->fmt_table.iommu.nid = NUMA_NO_NODE;

drivers/iommu/generic_pt/kunit_iommu.h

141

priv->fmt_table.iommu.driver_ops = &pt_kunit_driver_ops;

drivers/iommu/generic_pt/kunit_iommu.h

142

priv->fmt_table.iommu.iommu_device = priv->dummy_dev;

drivers/iommu/generic_pt/kunit_iommu.h

153

priv->iommu = &priv->fmt_table.iommu;

drivers/iommu/generic_pt/kunit_iommu.h

154

priv->common = common_from_iommu(&priv->fmt_table.iommu);

drivers/iommu/generic_pt/kunit_iommu.h

155

priv->iommu->ops->get_info(priv->iommu, &priv->info);

drivers/iommu/generic_pt/kunit_iommu.h

65

struct pt_iommu *iommu;

drivers/iommu/generic_pt/kunit_iommu.h

77

PT_IOMMU_CHECK_DOMAIN(struct kunit_iommu_priv, fmt_table.iommu, domain);

drivers/iommu/generic_pt/kunit_iommu.h

99

iommu_table, struct kunit_iommu_priv, fmt_table.iommu);

drivers/iommu/generic_pt/kunit_iommu_pt.h

467

pt_iommu_deinit(priv->iommu);

drivers/iommu/intel/cache.c

104

if (cache_tage_match(tag, did, iommu, dev, pasid, type)) {

drivers/iommu/intel/cache.c

202

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/cache.c

211

return domain_id_iommu(domain, iommu);

drivers/iommu/intel/cache.c

24

struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/cache.c

293

static void qi_batch_flush_descs(struct intel_iommu *iommu, struct qi_batch *batch)

drivers/iommu/intel/cache.c

295

if (!iommu || !batch->index)

drivers/iommu/intel/cache.c

298

qi_submit_sync(iommu, batch->descs, batch->index, 0);

drivers/iommu/intel/cache.c

304

static void qi_batch_increment_index(struct intel_iommu *iommu, struct qi_batch *batch)

drivers/iommu/intel/cache.c

307

qi_batch_flush_descs(iommu, batch);

drivers/iommu/intel/cache.c

310

static void qi_batch_add_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/cache.c

314

qi_desc_iotlb(iommu, did, addr, size_order, type, &batch->descs[batch->index]);

drivers/iommu/intel/cache.c

315

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

318

static void qi_batch_add_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/cache.c

326

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/cache.c

330

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

333

static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,

drivers/iommu/intel/cache.c

34

return tag->iommu == iommu;

drivers/iommu/intel/cache.c

346

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

349

static void qi_batch_add_pasid_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/cache.c

358

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/cache.c

363

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

377

struct intel_iommu *iommu = tag->iommu;

drivers/iommu/intel/cache.c

381

qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,

drivers/iommu/intel/cache.c

390

if (!cap_pgsel_inv(iommu->cap) ||

drivers/iommu/intel/cache.c

391

mask > cap_max_amask_val(iommu->cap) || pages == -1) {

drivers/iommu/intel/cache.c

398

if (ecap_qis(iommu->ecap))

drivers/iommu/intel/cache.c

399

qi_batch_add_iotlb(iommu, tag->domain_id, addr | ih, mask, type,

drivers/iommu/intel/cache.c

402

__iommu_flush_iotlb(iommu, tag->domain_id, addr | ih, mask, type);

drivers/iommu/intel/cache.c

408

struct intel_iommu *iommu = tag->iommu;

drivers/iommu/intel/cache.c

416

qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep,

drivers/iommu/intel/cache.c

419

qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep,

drivers/iommu/intel/cache.c

424

qi_batch_add_pasid_dev_iotlb(iommu, sid, info->pfsid, tag->pasid,

drivers/iommu/intel/cache.c

427

qi_batch_add_pasid_dev_iotlb(iommu, sid, info->pfsid, tag->pasid,

drivers/iommu/intel/cache.c

439

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/cache.c

454

if (iommu && iommu != tag->iommu)

drivers/iommu/intel/cache.c

455

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

456

iommu = tag->iommu;

drivers/iommu/intel/cache.c

47

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/cache.c

481

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

508

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/cache.c

517

if (iommu && iommu != tag->iommu)

drivers/iommu/intel/cache.c

518

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

519

iommu = tag->iommu;

drivers/iommu/intel/cache.c

521

if (!cap_caching_mode(iommu->cap) ||

drivers/iommu/intel/cache.c

523

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/cache.c

533

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

57

tag->iommu = iommu;

drivers/iommu/intel/cache.c

65

tag->dev = iommu->iommu.dev;

drivers/iommu/intel/cache.c

70

if (cache_tage_match(temp, did, iommu, dev, pasid, type)) {

drivers/iommu/intel/cache.c

77

if (temp->iommu == iommu)

drivers/iommu/intel/cache.c

98

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/debugfs.c

114

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

120

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

128

iommu->name, drhd->reg_base_addr);

drivers/iommu/intel/debugfs.c

134

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/debugfs.c

136

value = dmar_readl(iommu->reg + iommu_regs_32[i].offset);

drivers/iommu/intel/debugfs.c

142

value = dmar_readq(iommu->reg + iommu_regs_64[i].offset);

drivers/iommu/intel/debugfs.c

147

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/debugfs.c

215

static void ctx_tbl_walk(struct seq_file *m, struct intel_iommu *iommu, u16 bus)

drivers/iommu/intel/debugfs.c

237

context = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/debugfs.c

246

tbl_wlk.rt_entry = &iommu->root_entry[bus];

drivers/iommu/intel/debugfs.c

250

if (dmar_readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT) {

drivers/iommu/intel/debugfs.c

261

static void root_tbl_walk(struct seq_file *m, struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

265

spin_lock(&iommu->lock);

drivers/iommu/intel/debugfs.c

266

seq_printf(m, "IOMMU %s: Root Table Address: 0x%llx\n", iommu->name,

drivers/iommu/intel/debugfs.c

267

(u64)virt_to_phys(iommu->root_entry));

drivers/iommu/intel/debugfs.c

276

ctx_tbl_walk(m, iommu, bus);

drivers/iommu/intel/debugfs.c

277

spin_unlock(&iommu->lock);

drivers/iommu/intel/debugfs.c

283

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

287

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

288

sts = dmar_readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/debugfs.c

291

iommu->name);

drivers/iommu/intel/debugfs.c

294

root_tbl_walk(m, iommu);

drivers/iommu/intel/debugfs.c

351

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

359

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

364

if (seg != iommu->segment)

drivers/iommu/intel/debugfs.c

367

sts = dmar_readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/debugfs.c

370

iommu->name);

drivers/iommu/intel/debugfs.c

373

if (dmar_readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT)

drivers/iommu/intel/debugfs.c

387

spin_lock(&iommu->lock);

drivers/iommu/intel/debugfs.c

389

context = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/debugfs.c

454

iommu->segment, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));

drivers/iommu/intel/debugfs.c

467

spin_unlock(&iommu->lock);

drivers/iommu/intel/debugfs.c

494

struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

496

int index, shift = qi_shift(iommu);

drivers/iommu/intel/debugfs.c

500

if (ecap_smts(iommu->ecap))

drivers/iommu/intel/debugfs.c

507

desc = iommu->qi->desc + offset;

drivers/iommu/intel/debugfs.c

508

if (ecap_smts(iommu->ecap))

drivers/iommu/intel/debugfs.c

512

iommu->qi->desc_status[index]);

drivers/iommu/intel/debugfs.c

516

iommu->qi->desc_status[index]);

drivers/iommu/intel/debugfs.c

523

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

529

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

530

qi = iommu->qi;

drivers/iommu/intel/debugfs.c

531

shift = qi_shift(iommu);

drivers/iommu/intel/debugfs.c

533

if (!qi || !ecap_qis(iommu->ecap))

drivers/iommu/intel/debugfs.c

536

seq_printf(m, "Invalidation queue on IOMMU: %s\n", iommu->name);

drivers/iommu/intel/debugfs.c

541

dmar_readq(iommu->reg + DMAR_IQH_REG) >> shift,

drivers/iommu/intel/debugfs.c

542

dmar_readq(iommu->reg + DMAR_IQT_REG) >> shift);

drivers/iommu/intel/debugfs.c

543

invalidation_queue_entry_show(m, iommu);

drivers/iommu/intel/debugfs.c

555

struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

565

ri_entry = &iommu->ir_table->base[idx];

drivers/iommu/intel/debugfs.c

579

struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

589

pi_entry = &iommu->ir_table->base[idx];

drivers/iommu/intel/debugfs.c

611

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

616

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

617

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/debugfs.c

621

iommu->name);

drivers/iommu/intel/debugfs.c

623

sts = dmar_readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/debugfs.c

624

if (iommu->ir_table && (sts & DMA_GSTS_IRES)) {

drivers/iommu/intel/debugfs.c

625

irta = virt_to_phys(iommu->ir_table->base);

drivers/iommu/intel/debugfs.c

627

ir_tbl_remap_entry_show(m, iommu);

drivers/iommu/intel/debugfs.c

636

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

637

if (!cap_pi_support(iommu->cap))

drivers/iommu/intel/debugfs.c

641

iommu->name);

drivers/iommu/intel/debugfs.c

643

if (iommu->ir_table) {

drivers/iommu/intel/debugfs.c

644

irta = virt_to_phys(iommu->ir_table->base);

drivers/iommu/intel/debugfs.c

646

ir_tbl_posted_entry_show(m, iommu);

drivers/iommu/intel/debugfs.c

659

static void latency_show_one(struct seq_file *m, struct intel_iommu *iommu,

drivers/iommu/intel/debugfs.c

663

iommu->name, drhd->reg_base_addr);

drivers/iommu/intel/debugfs.c

665

dmar_latency_snapshot(iommu, debug_buf, DEBUG_BUFFER_SIZE);

drivers/iommu/intel/debugfs.c

672

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

675

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

676

latency_show_one(m, iommu, drhd);

drivers/iommu/intel/debugfs.c

692

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

710

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

711

dmar_latency_disable(iommu, DMAR_LATENCY_INV_IOTLB);

drivers/iommu/intel/debugfs.c

712

dmar_latency_disable(iommu, DMAR_LATENCY_INV_DEVTLB);

drivers/iommu/intel/debugfs.c

713

dmar_latency_disable(iommu, DMAR_LATENCY_INV_IEC);

drivers/iommu/intel/debugfs.c

719

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

720

dmar_latency_enable(iommu, DMAR_LATENCY_INV_IOTLB);

drivers/iommu/intel/debugfs.c

725

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

726

dmar_latency_enable(iommu, DMAR_LATENCY_INV_DEVTLB);

drivers/iommu/intel/debugfs.c

731

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

732

dmar_latency_enable(iommu, DMAR_LATENCY_INV_IEC);

drivers/iommu/intel/dmar.c

1000

release_mem_region(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

1001

iommu->reg_size = map_size;

drivers/iommu/intel/dmar.c

1002

if (!request_mem_region(iommu->reg_phys, iommu->reg_size,

drivers/iommu/intel/dmar.c

1003

iommu->name)) {

drivers/iommu/intel/dmar.c

1008

iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

1009

if (!iommu->reg) {

drivers/iommu/intel/dmar.c

1016

if (cap_ecmds(iommu->cap)) {

drivers/iommu/intel/dmar.c

1020

iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG +

drivers/iommu/intel/dmar.c

1029

iounmap(iommu->reg);

drivers/iommu/intel/dmar.c

1031

release_mem_region(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

1038

struct intel_iommu *iommu;

drivers/iommu/intel/dmar.c

1049

iommu = kzalloc_obj(*iommu);

drivers/iommu/intel/dmar.c

1050

if (!iommu)

drivers/iommu/intel/dmar.c

1053

iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,

drivers/iommu/intel/dmar.c

1055

if (iommu->seq_id < 0) {

drivers/iommu/intel/dmar.c

1057

err = iommu->seq_id;

drivers/iommu/intel/dmar.c

1060

snprintf(iommu->name, sizeof(iommu->name), "dmar%d", iommu->seq_id);

drivers/iommu/intel/dmar.c

1062

err = map_iommu(iommu, drhd);

drivers/iommu/intel/dmar.c

1064

pr_err("Failed to map %s\n", iommu->name);

drivers/iommu/intel/dmar.c

1068

if (!cap_sagaw(iommu->cap) &&

drivers/iommu/intel/dmar.c

1069

(!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) {

drivers/iommu/intel/dmar.c

1071

iommu->name);

drivers/iommu/intel/dmar.c

1076

agaw = iommu_calculate_agaw(iommu);

drivers/iommu/intel/dmar.c

1079

iommu->seq_id);

drivers/iommu/intel/dmar.c

1084

msagaw = iommu_calculate_max_sagaw(iommu);

drivers/iommu/intel/dmar.c

1087

iommu->seq_id);

drivers/iommu/intel/dmar.c

1092

iommu->agaw = agaw;

drivers/iommu/intel/dmar.c

1093

iommu->msagaw = msagaw;

drivers/iommu/intel/dmar.c

1094

iommu->segment = drhd->segment;

drivers/iommu/intel/dmar.c

1095

iommu->device_rbtree = RB_ROOT;

drivers/iommu/intel/dmar.c

1096

spin_lock_init(&iommu->device_rbtree_lock);

drivers/iommu/intel/dmar.c

1097

mutex_init(&iommu->iopf_lock);

drivers/iommu/intel/dmar.c

1098

iommu->node = NUMA_NO_NODE;

drivers/iommu/intel/dmar.c

1099

spin_lock_init(&iommu->lock);

drivers/iommu/intel/dmar.c

1100

ida_init(&iommu->domain_ida);

drivers/iommu/intel/dmar.c

1101

mutex_init(&iommu->did_lock);

drivers/iommu/intel/dmar.c

1103

ver = readl(iommu->reg + DMAR_VER_REG);

drivers/iommu/intel/dmar.c

1105

iommu->name,

drivers/iommu/intel/dmar.c

1108

(unsigned long long)iommu->cap,

drivers/iommu/intel/dmar.c

1109

(unsigned long long)iommu->ecap);

drivers/iommu/intel/dmar.c

1112

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/dmar.c

1114

iommu->gcmd |= DMA_GCMD_IRE;

drivers/iommu/intel/dmar.c

1116

iommu->gcmd |= DMA_GCMD_TE;

drivers/iommu/intel/dmar.c

1118

iommu->gcmd |= DMA_GCMD_QIE;

drivers/iommu/intel/dmar.c

1120

if (alloc_iommu_pmu(iommu))

drivers/iommu/intel/dmar.c

1121

pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id);

drivers/iommu/intel/dmar.c

1123

raw_spin_lock_init(&iommu->register_lock);

drivers/iommu/intel/dmar.c

1129

if (pasid_supported(iommu))

drivers/iommu/intel/dmar.c

1130

iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);

drivers/iommu/intel/dmar.c

1138

err = iommu_device_sysfs_add(&iommu->iommu, NULL,

drivers/iommu/intel/dmar.c

1140

"%s", iommu->name);

drivers/iommu/intel/dmar.c

1144

err = iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);

drivers/iommu/intel/dmar.c

1148

iommu_pmu_register(iommu);

drivers/iommu/intel/dmar.c

1151

drhd->iommu = iommu;

drivers/iommu/intel/dmar.c

1152

iommu->drhd = drhd;

drivers/iommu/intel/dmar.c

1157

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/intel/dmar.c

1159

free_iommu_pmu(iommu);

drivers/iommu/intel/dmar.c

1160

unmap_iommu(iommu);

drivers/iommu/intel/dmar.c

1162

ida_free(&dmar_seq_ids, iommu->seq_id);

drivers/iommu/intel/dmar.c

1164

kfree(iommu);

drivers/iommu/intel/dmar.c

1168

static void free_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1170

if (intel_iommu_enabled && !iommu->drhd->ignored) {

drivers/iommu/intel/dmar.c

1171

iommu_pmu_unregister(iommu);

drivers/iommu/intel/dmar.c

1172

iommu_device_unregister(&iommu->iommu);

drivers/iommu/intel/dmar.c

1173

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/intel/dmar.c

1176

free_iommu_pmu(iommu);

drivers/iommu/intel/dmar.c

1178

if (iommu->irq) {

drivers/iommu/intel/dmar.c

1179

if (iommu->pr_irq) {

drivers/iommu/intel/dmar.c

1180

free_irq(iommu->pr_irq, iommu);

drivers/iommu/intel/dmar.c

1181

dmar_free_hwirq(iommu->pr_irq);

drivers/iommu/intel/dmar.c

1182

iommu->pr_irq = 0;

drivers/iommu/intel/dmar.c

1184

free_irq(iommu->irq, iommu);

drivers/iommu/intel/dmar.c

1185

dmar_free_hwirq(iommu->irq);

drivers/iommu/intel/dmar.c

1186

iommu->irq = 0;

drivers/iommu/intel/dmar.c

1189

if (iommu->qi) {

drivers/iommu/intel/dmar.c

1190

iommu_free_pages(iommu->qi->desc);

drivers/iommu/intel/dmar.c

1191

kfree(iommu->qi->desc_status);

drivers/iommu/intel/dmar.c

1192

kfree(iommu->qi);

drivers/iommu/intel/dmar.c

1195

if (iommu->reg)

drivers/iommu/intel/dmar.c

1196

unmap_iommu(iommu);

drivers/iommu/intel/dmar.c

1198

ida_destroy(&iommu->domain_ida);

drivers/iommu/intel/dmar.c

1199

ida_free(&dmar_seq_ids, iommu->seq_id);

drivers/iommu/intel/dmar.c

1200

kfree(iommu);

drivers/iommu/intel/dmar.c

1240

static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)

drivers/iommu/intel/dmar.c

1242

unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG);

drivers/iommu/intel/dmar.c

1243

u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);

drivers/iommu/intel/dmar.c

1244

struct qi_desc *desc = iommu->qi->desc + head;

drivers/iommu/intel/dmar.c

1261

head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH;

drivers/iommu/intel/dmar.c

1262

head <<= qi_shift(iommu);

drivers/iommu/intel/dmar.c

1263

desc = iommu->qi->desc + head;

drivers/iommu/intel/dmar.c

1271

static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)

drivers/iommu/intel/dmar.c

1277

struct q_inval *qi = iommu->qi;

drivers/iommu/intel/dmar.c

1278

int shift = qi_shift(iommu);

drivers/iommu/intel/dmar.c

1283

fault = readl(iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1285

qi_dump_fault(iommu, fault);

drivers/iommu/intel/dmar.c

1293

head = readl(iommu->reg + DMAR_IQH_REG);

drivers/iommu/intel/dmar.c

1304

writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1315

head = readl(iommu->reg + DMAR_IQH_REG);

drivers/iommu/intel/dmar.c

1317

tail = readl(iommu->reg + DMAR_IQT_REG);

drivers/iommu/intel/dmar.c

1324

iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);

drivers/iommu/intel/dmar.c

1327

writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1344

dev = device_rbtree_find(iommu, ite_sid);

drivers/iommu/intel/dmar.c

1354

writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1368

int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,

drivers/iommu/intel/dmar.c

1371

struct q_inval *qi = iommu->qi;

drivers/iommu/intel/dmar.c

1388

dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IOTLB))

drivers/iommu/intel/dmar.c

1392

dmar_latency_enabled(iommu, DMAR_LATENCY_INV_DEVTLB))

drivers/iommu/intel/dmar.c

1396

dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IEC))

drivers/iommu/intel/dmar.c

1416

shift = qi_shift(iommu);

drivers/iommu/intel/dmar.c

1422

trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1,

drivers/iommu/intel/dmar.c

1445

writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);

drivers/iommu/intel/dmar.c

1455

rc = qi_check_fault(iommu, index, wait_index);

drivers/iommu/intel/dmar.c

1482

dmar_latency_update(iommu, DMAR_LATENCY_INV_IOTLB,

drivers/iommu/intel/dmar.c

1486

dmar_latency_update(iommu, DMAR_LATENCY_INV_DEVTLB,

drivers/iommu/intel/dmar.c

1490

dmar_latency_update(iommu, DMAR_LATENCY_INV_IEC,

drivers/iommu/intel/dmar.c

1499

void qi_global_iec(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1509

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1512

void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,

drivers/iommu/intel/dmar.c

1523

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1526

void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/dmar.c

1531

qi_desc_iotlb(iommu, did, addr, size_order, type, &desc);

drivers/iommu/intel/dmar.c

1532

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1535

void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/dmar.c

1546

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/dmar.c

1550

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1554

void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,

drivers/iommu/intel/dmar.c

1570

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1574

void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/dmar.c

1585

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/dmar.c

1591

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1594

void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,

drivers/iommu/intel/dmar.c

1601

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1607

void dmar_disable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1613

if (!ecap_qis(iommu->ecap))

drivers/iommu/intel/dmar.c

1616

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1618

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/dmar.c

1625

while ((readl(iommu->reg + DMAR_IQT_REG) !=

drivers/iommu/intel/dmar.c

1626

readl(iommu->reg + DMAR_IQH_REG)) &&

drivers/iommu/intel/dmar.c

1630

iommu->gcmd &= ~DMA_GCMD_QIE;

drivers/iommu/intel/dmar.c

1631

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/dmar.c

1633

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,

drivers/iommu/intel/dmar.c

1636

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1642

static void __dmar_enable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1646

struct q_inval *qi = iommu->qi;

drivers/iommu/intel/dmar.c

1656

if (ecap_smts(iommu->ecap))

drivers/iommu/intel/dmar.c

1659

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1662

writel(0, iommu->reg + DMAR_IQT_REG);

drivers/iommu/intel/dmar.c

1664

dmar_writeq(iommu->reg + DMAR_IQA_REG, val);

drivers/iommu/intel/dmar.c

1666

iommu->gcmd |= DMA_GCMD_QIE;

drivers/iommu/intel/dmar.c

1667

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/dmar.c

1670

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);

drivers/iommu/intel/dmar.c

1672

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1680

int dmar_enable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1685

if (!ecap_qis(iommu->ecap))

drivers/iommu/intel/dmar.c

1691

if (iommu->qi)

drivers/iommu/intel/dmar.c

1694

iommu->qi = kmalloc_obj(*qi, GFP_ATOMIC);

drivers/iommu/intel/dmar.c

1695

if (!iommu->qi)

drivers/iommu/intel/dmar.c

1698

qi = iommu->qi;

drivers/iommu/intel/dmar.c

1704

desc = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC,

drivers/iommu/intel/dmar.c

1705

ecap_smts(iommu->ecap) ? SZ_8K :

drivers/iommu/intel/dmar.c

1709

iommu->qi = NULL;

drivers/iommu/intel/dmar.c

1719

iommu->qi = NULL;

drivers/iommu/intel/dmar.c

1725

__dmar_enable_qi(iommu);

drivers/iommu/intel/dmar.c

1845

static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)

drivers/iommu/intel/dmar.c

1847

if (iommu->irq == irq)

drivers/iommu/intel/dmar.c

1849

else if (iommu->pr_irq == irq)

drivers/iommu/intel/dmar.c

1851

else if (iommu->perf_irq == irq)

drivers/iommu/intel/dmar.c

1859

struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);

drivers/iommu/intel/dmar.c

1860

int reg = dmar_msi_reg(iommu, data->irq);

drivers/iommu/intel/dmar.c

1864

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1865

writel(0, iommu->reg + reg);

drivers/iommu/intel/dmar.c

1867

readl(iommu->reg + reg);

drivers/iommu/intel/dmar.c

1868

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1873

struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);

drivers/iommu/intel/dmar.c

1874

int reg = dmar_msi_reg(iommu, data->irq);

drivers/iommu/intel/dmar.c

1878

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1879

writel(DMA_FECTL_IM, iommu->reg + reg);

drivers/iommu/intel/dmar.c

1881

readl(iommu->reg + reg);

drivers/iommu/intel/dmar.c

1882

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1887

struct intel_iommu *iommu = irq_get_handler_data(irq);

drivers/iommu/intel/dmar.c

1888

int reg = dmar_msi_reg(iommu, irq);

drivers/iommu/intel/dmar.c

1891

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1892

writel(msg->data, iommu->reg + reg + 4);

drivers/iommu/intel/dmar.c

1893

writel(msg->address_lo, iommu->reg + reg + 8);

drivers/iommu/intel/dmar.c

1894

writel(msg->address_hi, iommu->reg + reg + 12);

drivers/iommu/intel/dmar.c

1895

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1898

static int dmar_fault_do_one(struct intel_iommu *iommu, int type,

drivers/iommu/intel/dmar.c

1929

dmar_fault_dump_ptes(iommu, source_id, addr, pasid);

drivers/iommu/intel/dmar.c

1937

struct intel_iommu *iommu = dev_id;

drivers/iommu/intel/dmar.c

1945

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1946

fault_status = readl(iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1955

reg = cap_fault_reg_offset(iommu->cap);

drivers/iommu/intel/dmar.c

1968

data = readl(iommu->reg + reg +

drivers/iommu/intel/dmar.c

1978

data = readl(iommu->reg + reg +

drivers/iommu/intel/dmar.c

1983

guest_addr = dmar_readq(iommu->reg + reg +

drivers/iommu/intel/dmar.c

1989

writel(DMA_FRCD_F, iommu->reg + reg +

drivers/iommu/intel/dmar.c

1992

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1996

dmar_fault_do_one(iommu, type, fault_reason,

drivers/iommu/intel/dmar.c

2001

if (fault_index >= cap_num_fault_regs(iommu->cap))

drivers/iommu/intel/dmar.c

2003

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

2007

iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

2010

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

2014

int dmar_set_interrupt(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

2021

if (iommu->irq)

drivers/iommu/intel/dmar.c

2024

irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);

drivers/iommu/intel/dmar.c

2026

iommu->irq = irq;

drivers/iommu/intel/dmar.c

2032

ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);

drivers/iommu/intel/dmar.c

2041

struct intel_iommu *iommu;

drivers/iommu/intel/dmar.c

2047

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/dmar.c

2051

if (iommu->irq || iommu->node != cpu_to_node(cpu))

drivers/iommu/intel/dmar.c

2054

ret = dmar_set_interrupt(iommu);

drivers/iommu/intel/dmar.c

2065

dmar_fault(iommu->irq, iommu);

drivers/iommu/intel/dmar.c

2066

fault_status = readl(iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

2067

writel(fault_status, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

2076

int dmar_reenable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

2078

if (!ecap_qis(iommu->ecap))

drivers/iommu/intel/dmar.c

2081

if (!iommu->qi)

drivers/iommu/intel/dmar.c

2087

dmar_disable_qi(iommu);

drivers/iommu/intel/dmar.c

2093

__dmar_enable_qi(iommu);

drivers/iommu/intel/dmar.c

462

if (dmaru->iommu)

drivers/iommu/intel/dmar.c

463

free_iommu(dmaru->iommu);

drivers/iommu/intel/dmar.c

502

drhd->iommu->node = node;

drivers/iommu/intel/dmar.c

68

static void free_iommu(struct intel_iommu *iommu);

drivers/iommu/intel/dmar.c

939

x86_init.iommu.iommu_init = intel_iommu_init;

drivers/iommu/intel/dmar.c

950

static void unmap_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

952

iounmap(iommu->reg);

drivers/iommu/intel/dmar.c

953

release_mem_region(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

964

static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd)

drivers/iommu/intel/dmar.c

969

iommu->reg_phys = phys_addr;

drivers/iommu/intel/dmar.c

970

iommu->reg_size = drhd->reg_size;

drivers/iommu/intel/dmar.c

972

if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {

drivers/iommu/intel/dmar.c

978

iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

979

if (!iommu->reg) {

drivers/iommu/intel/dmar.c

985

iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);

drivers/iommu/intel/dmar.c

986

iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);

drivers/iommu/intel/dmar.c

988

if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {

drivers/iommu/intel/dmar.c

995

map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),

drivers/iommu/intel/dmar.c

996

cap_max_fault_reg_offset(iommu->cap));

drivers/iommu/intel/dmar.c

998

if (map_size > iommu->reg_size) {

drivers/iommu/intel/dmar.c

999

iounmap(iommu->reg);

drivers/iommu/intel/iommu.c

1002

free_context_table(iommu);

drivers/iommu/intel/iommu.c

1004

if (ecap_prs(iommu->ecap))

drivers/iommu/intel/iommu.c

1005

intel_iommu_finish_prq(iommu);

drivers/iommu/intel/iommu.c

1012

static bool first_level_by_default(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1015

if (!sm_supported(iommu))

drivers/iommu/intel/iommu.c

1019

if (ecap_flts(iommu->ecap) ^ ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

1020

return ecap_flts(iommu->ecap);

drivers/iommu/intel/iommu.c

1025

int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1037

guard(mutex)(&iommu->did_lock);

drivers/iommu/intel/iommu.c

1038

curr = xa_load(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.c

1045

num = ida_alloc_range(&iommu->domain_ida, IDA_START_DID,

drivers/iommu/intel/iommu.c

1046

cap_ndoms(iommu->cap) - 1, GFP_KERNEL);

drivers/iommu/intel/iommu.c

1048

pr_err("%s: No free domain ids\n", iommu->name);

drivers/iommu/intel/iommu.c

1054

info->iommu = iommu;

drivers/iommu/intel/iommu.c

1055

curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,

drivers/iommu/intel/iommu.c

1065

ida_free(&iommu->domain_ida, info->did);

drivers/iommu/intel/iommu.c

1071

void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1078

guard(mutex)(&iommu->did_lock);

drivers/iommu/intel/iommu.c

1079

info = xa_load(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.c

1081

ida_free(&iommu->domain_ida, info->did);

drivers/iommu/intel/iommu.c

1082

xa_erase(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.c

1096

static void copied_context_tear_down(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1102

if (!context_copied(iommu, bus, devfn))

drivers/iommu/intel/iommu.c

1105

assert_spin_locked(&iommu->lock);

drivers/iommu/intel/iommu.c

1110

if (did_old < cap_ndoms(iommu->cap)) {

drivers/iommu/intel/iommu.c

1111

iommu->flush.flush_context(iommu, did_old,

drivers/iommu/intel/iommu.c

1115

iommu->flush.flush_iotlb(iommu, did_old, 0, 0,

drivers/iommu/intel/iommu.c

1119

clear_context_copied(iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1128

static void context_present_cache_flush(struct intel_iommu *iommu, u16 did,

drivers/iommu/intel/iommu.c

1131

if (cap_caching_mode(iommu->cap)) {

drivers/iommu/intel/iommu.c

1132

iommu->flush.flush_context(iommu, 0,

drivers/iommu/intel/iommu.c

1136

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/iommu.c

1138

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1143

struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1147

domain_lookup_dev_info(domain, iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1148

u16 did = domain_id_iommu(domain, iommu);

drivers/iommu/intel/iommu.c

1162

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

1164

context = iommu_context_addr(iommu, bus, devfn, 1);

drivers/iommu/intel/iommu.c

1169

if (context_present(context) && !context_copied(iommu, bus, devfn))

drivers/iommu/intel/iommu.c

1172

copied_context_tear_down(iommu, context, bus, devfn);

drivers/iommu/intel/iommu.c

1186

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/iommu.c

1188

context_present_cache_flush(iommu, did, bus, devfn);

drivers/iommu/intel/iommu.c

1192

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

1201

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1204

return domain_context_mapping_one(domain, iommu,

drivers/iommu/intel/iommu.c

1212

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1217

return domain_context_mapping_one(domain, iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1231

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1235

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

1236

context = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/iommu.c

1238

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

124

struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid)

drivers/iommu/intel/iommu.c

1244

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/iommu.c

1245

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

1248

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/iommu.c

1251

int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/iommu.c

1256

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

1258

return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, did, flags);

drivers/iommu/intel/iommu.c

1261

static int domain_setup_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1267

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

1269

return intel_pasid_setup_second_level(iommu, domain, dev, pasid);

drivers/iommu/intel/iommu.c

1272

static int domain_setup_passthrough(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1277

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

1279

return intel_pasid_setup_pass_through(iommu, dev, pasid);

drivers/iommu/intel/iommu.c

1282

static int domain_setup_first_level(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

130

spin_lock_irqsave(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1304

return __domain_setup_first_level(iommu, dev, pasid,

drivers/iommu/intel/iommu.c

1305

domain_id_iommu(domain, iommu),

drivers/iommu/intel/iommu.c

131

node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key);

drivers/iommu/intel/iommu.c

1313

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1317

ret = domain_attach_iommu(domain, iommu);

drivers/iommu/intel/iommu.c

1330

if (!sm_supported(iommu))

drivers/iommu/intel/iommu.c

1333

ret = domain_setup_first_level(iommu, domain, dev,

drivers/iommu/intel/iommu.c

1336

ret = domain_setup_second_level(iommu, domain, dev,

drivers/iommu/intel/iommu.c

134

spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1387

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

139

static int device_rbtree_insert(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1393

if (!ecap_pass_through(iommu->ecap))

drivers/iommu/intel/iommu.c

1406

static void intel_iommu_init_qi(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1414

if (!iommu->qi) {

drivers/iommu/intel/iommu.c

1418

dmar_fault(-1, iommu);

drivers/iommu/intel/iommu.c

1423

dmar_disable_qi(iommu);

drivers/iommu/intel/iommu.c

1426

if (dmar_enable_qi(iommu)) {

drivers/iommu/intel/iommu.c

1430

iommu->flush.flush_context = __iommu_flush_context;

drivers/iommu/intel/iommu.c

1431

iommu->flush.flush_iotlb = __iommu_flush_iotlb;

drivers/iommu/intel/iommu.c

1433

iommu->name);

drivers/iommu/intel/iommu.c

1435

iommu->flush.flush_context = qi_flush_context;

drivers/iommu/intel/iommu.c

1436

iommu->flush.flush_iotlb = qi_flush_iotlb;

drivers/iommu/intel/iommu.c

1437

pr_info("%s: Using Queued invalidation\n", iommu->name);

drivers/iommu/intel/iommu.c

1441

static int copy_context_table(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

145

spin_lock_irqsave(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

146

curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp);

drivers/iommu/intel/iommu.c

1463

__iommu_flush_cache(iommu, new_ce,

drivers/iommu/intel/iommu.c

147

spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1493

new_ce = iommu_alloc_pages_node_sz(iommu->node,

drivers/iommu/intel/iommu.c

1508

if (did >= 0 && did < cap_ndoms(iommu->cap))

drivers/iommu/intel/iommu.c

1509

ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);

drivers/iommu/intel/iommu.c

1511

set_context_copied(iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1517

__iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);

drivers/iommu/intel/iommu.c

1526

static int copy_translation_tables(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1536

rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);

drivers/iommu/intel/iommu.c

1538

new_ext = !!sm_supported(iommu);

drivers/iommu/intel/iommu.c

1549

iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);

drivers/iommu/intel/iommu.c

1550

if (!iommu->copied_tables)

drivers/iommu/intel/iommu.c

156

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1569

ret = copy_context_table(iommu, &old_rt[bus],

drivers/iommu/intel/iommu.c

1573

iommu->name, bus);

drivers/iommu/intel/iommu.c

1578

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

1587

iommu->root_entry[bus].lo = val;

drivers/iommu/intel/iommu.c

159

spin_lock_irqsave(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1594

iommu->root_entry[bus].hi = val;

drivers/iommu/intel/iommu.c

1597

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

160

rb_erase(&info->node, &iommu->device_rbtree);

drivers/iommu/intel/iommu.c

1601

__iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);

drivers/iommu/intel/iommu.c

161

spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1614

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

1617

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1619

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1628

if (pasid_supported(iommu)) {

drivers/iommu/intel/iommu.c

1629

u32 temp = 2 << ecap_pss(iommu->ecap);

drivers/iommu/intel/iommu.c

1635

intel_iommu_init_qi(iommu);

drivers/iommu/intel/iommu.c

1636

init_translation_status(iommu);

drivers/iommu/intel/iommu.c

1638

if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {

drivers/iommu/intel/iommu.c

1639

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1640

clear_translation_pre_enabled(iommu);

drivers/iommu/intel/iommu.c

1642

iommu->name);

drivers/iommu/intel/iommu.c

1650

ret = iommu_alloc_root_entry(iommu);

drivers/iommu/intel/iommu.c

1654

if (translation_pre_enabled(iommu)) {

drivers/iommu/intel/iommu.c

1657

ret = copy_translation_tables(iommu);

drivers/iommu/intel/iommu.c

1669

iommu->name);

drivers/iommu/intel/iommu.c

1670

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1671

clear_translation_pre_enabled(iommu);

drivers/iommu/intel/iommu.c

1674

iommu->name);

drivers/iommu/intel/iommu.c

1678

intel_svm_check(iommu);

drivers/iommu/intel/iommu.c

1686

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1687

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1688

iommu_set_root_entry(iommu);

drivers/iommu/intel/iommu.c

1700

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1707

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

1711

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1713

if (ecap_prs(iommu->ecap)) {

drivers/iommu/intel/iommu.c

1719

ret = intel_iommu_enable_prq(iommu);

drivers/iommu/intel/iommu.c

1725

ret = dmar_set_interrupt(iommu);

drivers/iommu/intel/iommu.c

1733

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1734

disable_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

1735

free_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

1781

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

1784

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1785

if (iommu->qi) {

drivers/iommu/intel/iommu.c

1786

ret = dmar_reenable_qi(iommu);

drivers/iommu/intel/iommu.c

1792

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1799

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

1803

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1804

iommu_set_root_entry(iommu);

drivers/iommu/intel/iommu.c

1805

iommu_enable_translation(iommu);

drivers/iommu/intel/iommu.c

1806

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

1815

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

1817

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1818

iommu->flush.flush_context(iommu, 0, 0, 0,

drivers/iommu/intel/iommu.c

1820

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

drivers/iommu/intel/iommu.c

1828

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

1833

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1834

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1836

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

1838

iommu->iommu_state[SR_DMAR_FECTL_REG] =

drivers/iommu/intel/iommu.c

1839

readl(iommu->reg + DMAR_FECTL_REG);

drivers/iommu/intel/iommu.c

1840

iommu->iommu_state[SR_DMAR_FEDATA_REG] =

drivers/iommu/intel/iommu.c

1841

readl(iommu->reg + DMAR_FEDATA_REG);

drivers/iommu/intel/iommu.c

1842

iommu->iommu_state[SR_DMAR_FEADDR_REG] =

drivers/iommu/intel/iommu.c

1843

readl(iommu->reg + DMAR_FEADDR_REG);

drivers/iommu/intel/iommu.c

1844

iommu->iommu_state[SR_DMAR_FEUADDR_REG] =

drivers/iommu/intel/iommu.c

1845

readl(iommu->reg + DMAR_FEUADDR_REG);

drivers/iommu/intel/iommu.c

1847

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

185

struct intel_iommu *iommu; /* the corresponding iommu */

drivers/iommu/intel/iommu.c

1855

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

1866

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1868

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

1870

writel(iommu->iommu_state[SR_DMAR_FECTL_REG],

drivers/iommu/intel/iommu.c

1871

iommu->reg + DMAR_FECTL_REG);

drivers/iommu/intel/iommu.c

1872

writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],

drivers/iommu/intel/iommu.c

1873

iommu->reg + DMAR_FEDATA_REG);

drivers/iommu/intel/iommu.c

1874

writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],

drivers/iommu/intel/iommu.c

1875

iommu->reg + DMAR_FEADDR_REG);

drivers/iommu/intel/iommu.c

1876

writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],

drivers/iommu/intel/iommu.c

1877

iommu->reg + DMAR_FEUADDR_REG);

drivers/iommu/intel/iommu.c

1879

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

2108

struct intel_iommu *iommu = dmaru->iommu;

drivers/iommu/intel/iommu.c

2114

if (iommu->gcmd & DMA_GCMD_TE)

drivers/iommu/intel/iommu.c

2115

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

2117

ret = iommu_alloc_root_entry(iommu);

drivers/iommu/intel/iommu.c

2121

intel_svm_check(iommu);

drivers/iommu/intel/iommu.c

2128

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2132

intel_iommu_init_qi(iommu);

drivers/iommu/intel/iommu.c

2133

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

2135

if (ecap_prs(iommu->ecap)) {

drivers/iommu/intel/iommu.c

2136

ret = intel_iommu_enable_prq(iommu);

drivers/iommu/intel/iommu.c

214

static bool translation_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

2141

ret = dmar_set_interrupt(iommu);

drivers/iommu/intel/iommu.c

2145

iommu_set_root_entry(iommu);

drivers/iommu/intel/iommu.c

2146

iommu_enable_translation(iommu);

drivers/iommu/intel/iommu.c

2148

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2152

disable_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

2154

free_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

216

return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);

drivers/iommu/intel/iommu.c

2161

struct intel_iommu *iommu = dmaru->iommu;

drivers/iommu/intel/iommu.c

2165

if (iommu == NULL)

drivers/iommu/intel/iommu.c

2171

disable_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

2172

free_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

219

static void clear_translation_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

221

iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/intel/iommu.c

2224

static bool dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

224

static void init_translation_status(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

2245

return !(satcu->atc_required && !sm_supported(iommu));

drivers/iommu/intel/iommu.c

228

gsts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/iommu.c

230

iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/intel/iommu.c

2353

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

2356

for_each_iommu(iommu, drhd)

drivers/iommu/intel/iommu.c

2357

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

2363

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

2373

iommu = drhd->iommu;

drivers/iommu/intel/iommu.c

2376

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2379

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

2387

return container_of(iommu_dev, struct intel_iommu, iommu);

drivers/iommu/intel/iommu.c

2393

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2394

u32 ver = readl(iommu->reg + DMAR_VER_REG);

drivers/iommu/intel/iommu.c

2403

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2404

return sysfs_emit(buf, "%llx\n", iommu->reg_phys);

drivers/iommu/intel/iommu.c

2411

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2412

return sysfs_emit(buf, "%llx\n", iommu->cap);

drivers/iommu/intel/iommu.c

2419

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2420

return sysfs_emit(buf, "%llx\n", iommu->ecap);

drivers/iommu/intel/iommu.c

2427

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2428

return sysfs_emit(buf, "%ld\n", cap_ndoms(iommu->cap));

drivers/iommu/intel/iommu.c

2435

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2439

for (id = 0; id < cap_ndoms(iommu->cap); id++)

drivers/iommu/intel/iommu.c

2440

if (ida_exists(&iommu->domain_ida, id))

drivers/iommu/intel/iommu.c

2505

struct intel_iommu *iommu __maybe_unused;

drivers/iommu/intel/iommu.c

2509

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

2556

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

2601

for_each_iommu(iommu, drhd)

drivers/iommu/intel/iommu.c

2602

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2637

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

2645

if (cap_caching_mode(iommu->cap) &&

drivers/iommu/intel/iommu.c

2646

!first_level_by_default(iommu)) {

drivers/iommu/intel/iommu.c

2650

iommu_device_sysfs_add(&iommu->iommu, NULL,

drivers/iommu/intel/iommu.c

2652

"%s", iommu->name);

drivers/iommu/intel/iommu.c

2659

iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);

drivers/iommu/intel/iommu.c

2662

iommu_pmu_register(iommu);

drivers/iommu/intel/iommu.c

2669

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

2670

if (!drhd->ignored && !translation_pre_enabled(iommu))

drivers/iommu/intel/iommu.c

2671

iommu_enable_translation(iommu);

drivers/iommu/intel/iommu.c

2673

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2723

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

2734

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

2735

intel_pasid_tear_down_entry(iommu, dev,

drivers/iommu/intel/iommu.c

2751

domain_detach_iommu(info->domain, iommu);

drivers/iommu/intel/iommu.c

2798

static unsigned int compute_vasz_lg2_fs(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

2801

unsigned int mgaw = cap_mgaw(iommu->cap);

drivers/iommu/intel/iommu.c

2810

if (mgaw > 48 && cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/iommu.c

2822

struct intel_iommu *iommu, u32 flags)

drivers/iommu/intel/iommu.c

2832

if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))

drivers/iommu/intel/iommu.c

2840

compute_vasz_lg2_fs(iommu, &cfg.top_level);

drivers/iommu/intel/iommu.c

2845

if (!ecap_smpwc(iommu->ecap))

drivers/iommu/intel/iommu.c

2847

dmar_domain->iommu.iommu_device = dev;

drivers/iommu/intel/iommu.c

2848

dmar_domain->iommu.nid = dev_to_node(dev);

drivers/iommu/intel/iommu.c

285

static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

2855

if (rwbf_required(iommu))

drivers/iommu/intel/iommu.c

2864

if (!cap_fl1gp_support(iommu->cap))

drivers/iommu/intel/iommu.c

2872

static unsigned int compute_vasz_lg2_ss(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

2875

unsigned int sagaw = cap_sagaw(iommu->cap);

drivers/iommu/intel/iommu.c

2876

unsigned int mgaw = cap_mgaw(iommu->cap);

drivers/iommu/intel/iommu.c

289

fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);

drivers/iommu/intel/iommu.c

290

sl_sagaw = cap_sagaw(iommu->cap);

drivers/iommu/intel/iommu.c

2904

struct intel_iommu *iommu, u32 flags)

drivers/iommu/intel/iommu.c

2917

!nested_supported(iommu)) ||

drivers/iommu/intel/iommu.c

2919

!ssads_supported(iommu)))

drivers/iommu/intel/iommu.c

2923

if (sm_supported(iommu) && !ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

293

if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))

drivers/iommu/intel/iommu.c

2930

cfg.common.hw_max_vasz_lg2 = compute_vasz_lg2_ss(iommu, &cfg.top_level);

drivers/iommu/intel/iommu.c

2942

if (!iommu_paging_structure_coherency(iommu))

drivers/iommu/intel/iommu.c

2944

dmar_domain->iommu.iommu_device = dev;

drivers/iommu/intel/iommu.c

2945

dmar_domain->iommu.nid = dev_to_node(dev);

drivers/iommu/intel/iommu.c

2959

sslps = cap_super_page_val(iommu->cap);

drivers/iommu/intel/iommu.c

297

if (!ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

2972

if (rwbf_required(iommu) || cap_caching_mode(iommu->cap))

drivers/iommu/intel/iommu.c

2983

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

2990

domain = intel_iommu_domain_alloc_first_stage(dev, iommu, flags);

drivers/iommu/intel/iommu.c

2993

return intel_iommu_domain_alloc_second_stage(dev, iommu, flags);

drivers/iommu/intel/iommu.c

3007

pt_iommu_deinit(&dmar_domain->iommu);

drivers/iommu/intel/iommu.c

3014

struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3021

if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))

drivers/iommu/intel/iommu.c

3024

if (!ecap_smpwc(iommu->ecap) &&

drivers/iommu/intel/iommu.c

303

static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)

drivers/iommu/intel/iommu.c

3030

if (!cap_fl5lp_support(iommu->cap) &&

drivers/iommu/intel/iommu.c

3035

if (!cap_fl1gp_support(iommu->cap) &&

drivers/iommu/intel/iommu.c

3040

if ((rwbf_required(iommu)) && !dmar_domain->iotlb_sync_map)

drivers/iommu/intel/iommu.c

3048

struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3051

unsigned int sslps = cap_super_page_val(iommu->cap);

drivers/iommu/intel/iommu.c

3056

if (dmar_domain->domain.dirty_ops && !ssads_supported(iommu))

drivers/iommu/intel/iommu.c

3058

if (dmar_domain->nested_parent && !nested_supported(iommu))

drivers/iommu/intel/iommu.c

3062

if (sm_supported(iommu) && !ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

3065

if (!iommu_paging_structure_coherency(iommu) &&

drivers/iommu/intel/iommu.c

3071

if (cap_mgaw(iommu->cap) < vasz_lg2)

drivers/iommu/intel/iommu.c

3075

if (!(cap_sagaw(iommu->cap) & BIT(pt_info.aw)))

drivers/iommu/intel/iommu.c

308

sagaw = __iommu_calculate_sagaw(iommu);

drivers/iommu/intel/iommu.c

3085

if ((rwbf_required(iommu) || cap_caching_mode(iommu->cap)) &&

drivers/iommu/intel/iommu.c

3095

!ecap_sc_support(iommu->ecap))

drivers/iommu/intel/iommu.c

3104

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3108

ret = paging_domain_compatible_first_stage(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3110

ret = paging_domain_compatible_second_stage(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3116

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&

drivers/iommu/intel/iommu.c

3117

context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/iommu.c

3162

if (!ecap_sc_support(info->iommu->ecap)) {

drivers/iommu/intel/iommu.c

3186

intel_pasid_setup_page_snoop_control(info->iommu, info->dev,

drivers/iommu/intel/iommu.c

320

int iommu_calculate_max_sagaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

322

return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);

drivers/iommu/intel/iommu.c

3220

return ecap_sc_support(info->iommu->ecap);

drivers/iommu/intel/iommu.c

3222

return ssads_supported(info->iommu);

drivers/iommu/intel/iommu.c

3232

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

3236

iommu = device_lookup_iommu(dev, &bus, &devfn);

drivers/iommu/intel/iommu.c

3237

if (!iommu || !iommu->iommu.ops)

drivers/iommu/intel/iommu.c

3251

info->segment = iommu->segment;

drivers/iommu/intel/iommu.c

3255

info->iommu = iommu;

drivers/iommu/intel/iommu.c

3257

if (ecap_dev_iotlb_support(iommu->ecap) &&

drivers/iommu/intel/iommu.c

3259

dmar_ats_supported(pdev, iommu)) {

drivers/iommu/intel/iommu.c

3270

if (ecap_dit(iommu->ecap))

drivers/iommu/intel/iommu.c

3274

if (sm_supported(iommu)) {

drivers/iommu/intel/iommu.c

3275

if (pasid_supported(iommu)) {

drivers/iommu/intel/iommu.c

3282

if (info->ats_supported && ecap_prs(iommu->ecap) &&

drivers/iommu/intel/iommu.c

3283

ecap_pds(iommu->ecap) && pci_pri_supported(pdev))

drivers/iommu/intel/iommu.c

3291

ret = device_rbtree_insert(iommu, info);

drivers/iommu/intel/iommu.c

3296

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {

drivers/iommu/intel/iommu.c

330

int iommu_calculate_agaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3303

if (!context_copied(iommu, info->bus, info->devfn)) {

drivers/iommu/intel/iommu.c

3312

return &iommu->iommu;

drivers/iommu/intel/iommu.c

332

return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);

drivers/iommu/intel/iommu.c

3326

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3338

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {

drivers/iommu/intel/iommu.c

3342

u16 did = domain_id_iommu(info->domain, iommu);

drivers/iommu/intel/iommu.c

335

static bool iommu_paging_structure_coherency(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3355

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3365

mutex_lock(&iommu->iopf_lock);

drivers/iommu/intel/iommu.c

3368

mutex_unlock(&iommu->iopf_lock);

drivers/iommu/intel/iommu.c

337

return sm_supported(iommu) ?

drivers/iommu/intel/iommu.c

3370

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&

drivers/iommu/intel/iommu.c

3371

!context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/iommu.c

338

ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);

drivers/iommu/intel/iommu.c

341

struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,

drivers/iommu/intel/iommu.c

344

struct root_entry *root = &iommu->root_entry[bus];

drivers/iommu/intel/iommu.c

3448

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3461

ret = iopf_queue_add_device(iommu->iopf_queue, dev);

drivers/iommu/intel/iommu.c

3473

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3482

iopf_queue_remove_device(iommu->iopf_queue, dev);

drivers/iommu/intel/iommu.c

3489

return translation_pre_enabled(info->iommu) && !info->domain;

drivers/iommu/intel/iommu.c

352

if (!alloc && context_copied(iommu, bus, devfn))

drivers/iommu/intel/iommu.c

3525

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3548

domain_detach_iommu(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

356

if (sm_supported(iommu)) {

drivers/iommu/intel/iommu.c

3561

intel_pasid_tear_down_entry(info->iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

3574

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3583

ret = domain_attach_iommu(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3599

domain_detach_iommu(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3611

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3618

if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))

drivers/iommu/intel/iommu.c

3624

if (context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/iommu.c

3640

ret = domain_setup_first_level(iommu, dmar_domain,

drivers/iommu/intel/iommu.c

3643

ret = domain_setup_second_level(iommu, dmar_domain,

drivers/iommu/intel/iommu.c

3668

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3680

vtd->cap_reg = iommu->cap;

drivers/iommu/intel/iommu.c

3681

vtd->ecap_reg = iommu->ecap;

drivers/iommu/intel/iommu.c

3697

ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev,

drivers/iommu/intel/iommu.c

370

context = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC,

drivers/iommu/intel/iommu.c

375

__iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);

drivers/iommu/intel/iommu.c

3771

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3774

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

3775

context = iommu_context_addr(iommu, bus, devfn, 1);

drivers/iommu/intel/iommu.c

3777

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

378

__iommu_flush_cache(iommu, entry, sizeof(*entry));

drivers/iommu/intel/iommu.c

3781

if (context_present(context) && !context_copied(iommu, bus, devfn)) {

drivers/iommu/intel/iommu.c

3782

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

3786

copied_context_tear_down(iommu, context, bus, devfn);

drivers/iommu/intel/iommu.c

3794

context_set_address_width(context, iommu->msagaw);

drivers/iommu/intel/iommu.c

3798

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/iommu.c

3800

context_present_cache_flush(iommu, FLPT_DEFAULT_DID, bus, devfn);

drivers/iommu/intel/iommu.c

3801

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

3829

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3842

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

3843

ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);

drivers/iommu/intel/iommu.c

3858

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3861

if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))

drivers/iommu/intel/iommu.c

3868

ret = domain_setup_passthrough(iommu, dev, pasid, old);

drivers/iommu/intel/iommu.c

4155

qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,

drivers/iommu/intel/iommu.c

4158

qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,

drivers/iommu/intel/iommu.c

4177

int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob)

drivers/iommu/intel/iommu.c

4183

if (!cap_ecmds(iommu->cap))

drivers/iommu/intel/iommu.c

4186

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

4188

res = dmar_readq(iommu->reg + DMAR_ECRSP_REG);

drivers/iommu/intel/iommu.c

4201

dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob);

drivers/iommu/intel/iommu.c

4202

dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT));

drivers/iommu/intel/iommu.c

4204

IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, dmar_readq,

drivers/iommu/intel/iommu.c

4214

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

440

static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)

drivers/iommu/intel/iommu.c

442

if (!iommu || iommu->drhd->ignored)

drivers/iommu/intel/iommu.c

461

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

483

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

517

iommu = NULL;

drivers/iommu/intel/iommu.c

519

if (iommu_is_dummy(iommu, dev))

drivers/iommu/intel/iommu.c

520

iommu = NULL;

drivers/iommu/intel/iommu.c

524

return iommu;

drivers/iommu/intel/iommu.c

527

static void free_context_table(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

53

#define rwbf_required(iommu) (rwbf_quirk || cap_rwbf((iommu)->cap))

drivers/iommu/intel/iommu.c

532

if (!iommu->root_entry)

drivers/iommu/intel/iommu.c

536

context = iommu_context_addr(iommu, i, 0, 0);

drivers/iommu/intel/iommu.c

540

if (!sm_supported(iommu))

drivers/iommu/intel/iommu.c

543

context = iommu_context_addr(iommu, i, 0x80, 0);

drivers/iommu/intel/iommu.c

548

iommu_free_pages(iommu->root_entry);

drivers/iommu/intel/iommu.c

549

iommu->root_entry = NULL;

drivers/iommu/intel/iommu.c

553

static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn,

drivers/iommu/intel/iommu.c

578

void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,

drivers/iommu/intel/iommu.c

590

pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);

drivers/iommu/intel/iommu.c

593

if (!iommu->root_entry) {

drivers/iommu/intel/iommu.c

597

rt_entry = &iommu->root_entry[bus];

drivers/iommu/intel/iommu.c

599

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

606

ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/iommu.c

616

if (!sm_supported(iommu)) {

drivers/iommu/intel/iommu.c

667

pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level);

drivers/iommu/intel/iommu.c

672

static int iommu_alloc_root_entry(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

676

root = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC, SZ_4K);

drivers/iommu/intel/iommu.c

679

iommu->name);

drivers/iommu/intel/iommu.c

683

__iommu_flush_cache(iommu, root, ROOT_SIZE);

drivers/iommu/intel/iommu.c

684

iommu->root_entry = root;

drivers/iommu/intel/iommu.c

689

static void iommu_set_root_entry(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

695

addr = virt_to_phys(iommu->root_entry);

drivers/iommu/intel/iommu.c

696

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

699

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

700

dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);

drivers/iommu/intel/iommu.c

702

writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

705

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

708

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

714

if (cap_esrtps(iommu->cap))

drivers/iommu/intel/iommu.c

717

iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);

drivers/iommu/intel/iommu.c

718

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

719

qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);

drivers/iommu/intel/iommu.c

720

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

drivers/iommu/intel/iommu.c

723

void iommu_flush_write_buffer(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

728

if (!rwbf_quirk && !cap_rwbf(iommu->cap))

drivers/iommu/intel/iommu.c

731

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

732

writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

735

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

738

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

742

static void __iommu_flush_context(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

762

iommu->name, type);

drivers/iommu/intel/iommu.c

767

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

768

dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);

drivers/iommu/intel/iommu.c

771

IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,

drivers/iommu/intel/iommu.c

774

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

777

void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.c

780

int tlb_offset = ecap_iotlb_offset(iommu->ecap);

drivers/iommu/intel/iommu.c

799

iommu->name, type);

drivers/iommu/intel/iommu.c

803

if (cap_write_drain(iommu->cap))

drivers/iommu/intel/iommu.c

806

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

809

dmar_writeq(iommu->reg + tlb_offset, val_iva);

drivers/iommu/intel/iommu.c

810

dmar_writeq(iommu->reg + tlb_offset + 8, val);

drivers/iommu/intel/iommu.c

813

IOMMU_WAIT_OP(iommu, tlb_offset + 8,

drivers/iommu/intel/iommu.c

816

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

829

struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.c

836

if (info->iommu == iommu && info->bus == bus &&

drivers/iommu/intel/iommu.c

914

iopf_queue_remove_device(info->iommu->iopf_queue, info->dev);

drivers/iommu/intel/iommu.c

925

static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

930

if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))

drivers/iommu/intel/iommu.c

933

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

934

pmen = readl(iommu->reg + DMAR_PMEN_REG);

drivers/iommu/intel/iommu.c

936

writel(pmen, iommu->reg + DMAR_PMEN_REG);

drivers/iommu/intel/iommu.c

939

IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,

drivers/iommu/intel/iommu.c

942

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

945

static void iommu_enable_translation(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

950

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

951

iommu->gcmd |= DMA_GCMD_TE;

drivers/iommu/intel/iommu.c

952

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

955

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

958

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

961

static void iommu_disable_translation(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

966

if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&

drivers/iommu/intel/iommu.c

967

(cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))

drivers/iommu/intel/iommu.c

970

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

971

iommu->gcmd &= ~DMA_GCMD_TE;

drivers/iommu/intel/iommu.c

972

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

975

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

978

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

981

static void disable_dmar_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

987

if (WARN_ON(!ida_is_empty(&iommu->domain_ida)))

drivers/iommu/intel/iommu.c

990

if (iommu->gcmd & DMA_GCMD_TE)

drivers/iommu/intel/iommu.c

991

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

994

static void free_dmar_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

996

if (iommu->copied_tables) {

drivers/iommu/intel/iommu.c

997

bitmap_free(iommu->copied_tables);

drivers/iommu/intel/iommu.c

998

iommu->copied_tables = NULL;

drivers/iommu/intel/iommu.h

1045

static inline void qi_desc_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

1052

if (cap_write_drain(iommu->cap))

drivers/iommu/intel/iommu.h

1055

if (cap_read_drain(iommu->cap))

drivers/iommu/intel/iommu.h

1159

int dmar_enable_qi(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1160

void dmar_disable_qi(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1161

int dmar_reenable_qi(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1162

void qi_global_iec(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1164

void qi_flush_context(struct intel_iommu *iommu, u16 did,

drivers/iommu/intel/iommu.h

1166

void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

1168

void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/iommu.h

1171

void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,

drivers/iommu/intel/iommu.h

1174

void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/iommu.h

1180

void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu,

drivers/iommu/intel/iommu.h

1183

int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,

drivers/iommu/intel/iommu.h

1186

void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

1194

int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1195

void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1205

int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/iommu.h

1211

void iommu_flush_write_buffer(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1216

struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid);

drivers/iommu/intel/iommu.h

1228

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.h

1256

int intel_iommu_enable_prq(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1257

int intel_iommu_finish_prq(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1299

void intel_svm_check(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1303

static inline void intel_svm_check(struct intel_iommu *iommu) {}

drivers/iommu/intel/iommu.h

1326

struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,

drivers/iommu/intel/iommu.h

1345

int iommu_calculate_agaw(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1346

int iommu_calculate_max_sagaw(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1347

int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob);

drivers/iommu/intel/iommu.h

1349

static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

1351

return (iommu->ecmdcap[DMA_ECMD_ECCAP3] & DMA_ECMD_ECCAP3_ESSENTIAL) ==

drivers/iommu/intel/iommu.h

1358

static inline int iommu_calculate_agaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

1362

static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

363

#define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \

drivers/iommu/intel/iommu.h

367

sts = op(iommu->reg + offset); \

drivers/iommu/intel/iommu.h

472

#define qi_shift(iommu) (DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap))

drivers/iommu/intel/iommu.h

518

void (*flush_context)(struct intel_iommu *iommu, u16 did, u16 sid,

drivers/iommu/intel/iommu.h

520

void (*flush_iotlb)(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

536

#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))

drivers/iommu/intel/iommu.h

537

#define pasid_supported(iommu) (sm_supported(iommu) && \

drivers/iommu/intel/iommu.h

538

ecap_pasid((iommu)->ecap))

drivers/iommu/intel/iommu.h

539

#define ssads_supported(iommu) (sm_supported(iommu) && \

drivers/iommu/intel/iommu.h

540

ecap_slads((iommu)->ecap) && \

drivers/iommu/intel/iommu.h

541

ecap_smpwc(iommu->ecap))

drivers/iommu/intel/iommu.h

542

#define nested_supported(iommu) (sm_supported(iommu) && \

drivers/iommu/intel/iommu.h

543

ecap_nest((iommu)->ecap))

drivers/iommu/intel/iommu.h

577

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.h

600

struct pt_iommu iommu;

drivers/iommu/intel/iommu.h

649

PT_IOMMU_CHECK_DOMAIN(struct dmar_domain, iommu, domain);

drivers/iommu/intel/iommu.h

650

PT_IOMMU_CHECK_DOMAIN(struct dmar_domain, sspt.iommu, domain);

drivers/iommu/intel/iommu.h

651

PT_IOMMU_CHECK_DOMAIN(struct dmar_domain, fspt.iommu, domain);

drivers/iommu/intel/iommu.h

664

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.h

734

struct iommu_device iommu; /* IOMMU core code handle */

drivers/iommu/intel/iommu.h

762

struct intel_iommu *iommu; /* IOMMU used by this device */

drivers/iommu/intel/iommu.h

782

struct intel_iommu *iommu, void *addr, int size)

drivers/iommu/intel/iommu.h

784

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/iommu.h

814

domain_id_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

817

xa_load(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.h

823

iommu_domain_did(struct iommu_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

828

return domain_id_iommu(to_dmar_domain(domain), iommu);

drivers/iommu/intel/iommu.h

973

static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.h

975

if (!iommu->copied_tables)

drivers/iommu/intel/iommu.h

978

return test_bit(((long)bus << 8) | devfn, iommu->copied_tables);

drivers/iommu/intel/iommu.h

982

set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.h

984

set_bit(((long)bus << 8) | devfn, iommu->copied_tables);

drivers/iommu/intel/iommu.h

988

clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.h

990

clear_bit(((long)bus << 8) | devfn, iommu->copied_tables);

drivers/iommu/intel/irq_remapping.c

101

iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;

drivers/iommu/intel/irq_remapping.c

1013

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/irq_remapping.c

1018

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

1019

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

1022

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1036

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/irq_remapping.c

1038

for_each_iommu(iommu, drhd)

drivers/iommu/intel/irq_remapping.c

1039

if (iommu->qi)

drivers/iommu/intel/irq_remapping.c

104

static int alloc_irte(struct intel_iommu *iommu,

drivers/iommu/intel/irq_remapping.c

1040

dmar_reenable_qi(iommu);

drivers/iommu/intel/irq_remapping.c

1045

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

1046

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

1050

iommu_set_irq_remapping(iommu, eim);

drivers/iommu/intel/irq_remapping.c

1051

iommu_enable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

107

struct ir_table *table = iommu->ir_table;

drivers/iommu/intel/irq_remapping.c

120

if (mask > ecap_max_handle_mask(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

123

ecap_max_handle_mask(iommu->ecap));

drivers/iommu/intel/irq_remapping.c

131

pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

133

irq_iommu->iommu = iommu;

drivers/iommu/intel/irq_remapping.c

1412

struct intel_iommu *iommu = domain->host_data;

drivers/iommu/intel/irq_remapping.c

1419

if (!info || !iommu)

drivers/iommu/intel/irq_remapping.c

143

static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)

drivers/iommu/intel/irq_remapping.c

1433

index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs);

drivers/iommu/intel/irq_remapping.c

1511

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/irq_remapping.c

1514

iommu = map_ioapic_to_iommu(fwspec->param[0]);

drivers/iommu/intel/irq_remapping.c

1516

iommu = map_hpet_to_iommu(fwspec->param[0]);

drivers/iommu/intel/irq_remapping.c

1518

return iommu && d == iommu->ir_domain;

drivers/iommu/intel/irq_remapping.c

153

return qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/irq_remapping.c

1540

static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

1545

if (eim && !ecap_eim_support(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

1547

iommu->reg_phys, iommu->ecap);

drivers/iommu/intel/irq_remapping.c

1551

if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) {

drivers/iommu/intel/irq_remapping.c

1553

iommu->reg_phys);

drivers/iommu/intel/irq_remapping.c

1560

ret = intel_setup_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1563

iommu->name);

drivers/iommu/intel/irq_remapping.c

1564

intel_teardown_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1565

ir_remove_ioapic_hpet_scope(iommu);

drivers/iommu/intel/irq_remapping.c

1567

iommu_enable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1576

struct intel_iommu *iommu = dmaru->iommu;

drivers/iommu/intel/irq_remapping.c

1580

if (iommu == NULL)

drivers/iommu/intel/irq_remapping.c

1582

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

1585

!cap_pi_support(iommu->cap))

drivers/iommu/intel/irq_remapping.c

1589

if (!iommu->ir_table)

drivers/iommu/intel/irq_remapping.c

159

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

1590

ret = dmar_ir_add(dmaru, iommu);

drivers/iommu/intel/irq_remapping.c

1592

if (iommu->ir_table) {

drivers/iommu/intel/irq_remapping.c

1593

if (!bitmap_empty(iommu->ir_table->bitmap,

drivers/iommu/intel/irq_remapping.c

1597

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1598

intel_teardown_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1599

ir_remove_ioapic_hpet_scope(iommu);

drivers/iommu/intel/irq_remapping.c

169

iommu = irq_iommu->iommu;

drivers/iommu/intel/irq_remapping.c

172

irte = &iommu->ir_table->base[index];

drivers/iommu/intel/irq_remapping.c

187

__iommu_flush_cache(iommu, irte, sizeof(*irte));

drivers/iommu/intel/irq_remapping.c

189

rc = qi_flush_iec(iommu, index, 0);

drivers/iommu/intel/irq_remapping.c

201

if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)

drivers/iommu/intel/irq_remapping.c

202

return ir_hpet[i].iommu;

drivers/iommu/intel/irq_remapping.c

212

if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)

drivers/iommu/intel/irq_remapping.c

213

return ir_ioapic[i].iommu;

drivers/iommu/intel/irq_remapping.c

222

return drhd ? drhd->iommu->ir_domain : NULL;

drivers/iommu/intel/irq_remapping.c

228

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

234

iommu = irq_iommu->iommu;

drivers/iommu/intel/irq_remapping.c

237

start = iommu->ir_table->base + index;

drivers/iommu/intel/irq_remapping.c

244

bitmap_release_region(iommu->ir_table->bitmap, index,

drivers/iommu/intel/irq_remapping.c

247

return qi_flush_iec(iommu, index, irq_iommu->irte_mask);

drivers/iommu/intel/irq_remapping.c

30

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

306

if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) {

drivers/iommu/intel/irq_remapping.c

331

if (ir_hpet[i].iommu && ir_hpet[i].id == id) {

drivers/iommu/intel/irq_remapping.c

37

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

416

static int iommu_load_old_irte(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

425

irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);

drivers/iommu/intel/irq_remapping.c

439

memcpy(iommu->ir_table->base, old_ir_table, size);

drivers/iommu/intel/irq_remapping.c

44

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

441

__iommu_flush_cache(iommu, iommu->ir_table->base, size);

drivers/iommu/intel/irq_remapping.c

448

if (iommu->ir_table->base[i].present)

drivers/iommu/intel/irq_remapping.c

449

bitmap_set(iommu->ir_table->bitmap, i, 1);

drivers/iommu/intel/irq_remapping.c

458

static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)

drivers/iommu/intel/irq_remapping.c

464

addr = virt_to_phys((void *)iommu->ir_table->base);

drivers/iommu/intel/irq_remapping.c

466

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

468

dmar_writeq(iommu->reg + DMAR_IRTA_REG,

drivers/iommu/intel/irq_remapping.c

472

writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

474

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

476

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

482

if (!cap_esirtps(iommu->cap))

drivers/iommu/intel/irq_remapping.c

483

qi_global_iec(iommu);

drivers/iommu/intel/irq_remapping.c

486

static void iommu_enable_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

491

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

494

iommu->gcmd |= DMA_GCMD_IRE;

drivers/iommu/intel/irq_remapping.c

495

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

496

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

501

iommu->gcmd &= ~DMA_GCMD_CFI;

drivers/iommu/intel/irq_remapping.c

502

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

503

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

517

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

520

static int intel_setup_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

527

.host_data = iommu,

drivers/iommu/intel/irq_remapping.c

533

if (iommu->ir_table)

drivers/iommu/intel/irq_remapping.c

542

iommu_alloc_pages_node_sz(iommu->node, GFP_KERNEL, SZ_1M);

drivers/iommu/intel/irq_remapping.c

544

pr_err("IR%d: failed to allocate 1M of pages\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

550

pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

554

info.fwnode = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

558

iommu->ir_domain = msi_create_parent_irq_domain(&info, &dmar_msi_parent_ops);

drivers/iommu/intel/irq_remapping.c

559

if (!iommu->ir_domain) {

drivers/iommu/intel/irq_remapping.c

560

pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

566

iommu->ir_table = ir_table;

drivers/iommu/intel/irq_remapping.c

572

if (!iommu->qi) {

drivers/iommu/intel/irq_remapping.c

576

dmar_fault(-1, iommu);

drivers/iommu/intel/irq_remapping.c

577

dmar_disable_qi(iommu);

drivers/iommu/intel/irq_remapping.c

579

if (dmar_enable_qi(iommu)) {

drivers/iommu/intel/irq_remapping.c

585

init_ir_status(iommu);

drivers/iommu/intel/irq_remapping.c

587

if (ir_pre_enabled(iommu)) {

drivers/iommu/intel/irq_remapping.c

590

iommu->name);

drivers/iommu/intel/irq_remapping.c

591

clear_ir_pre_enabled(iommu);

drivers/iommu/intel/irq_remapping.c

592

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

593

} else if (iommu_load_old_irte(iommu))

drivers/iommu/intel/irq_remapping.c

595

iommu->name);

drivers/iommu/intel/irq_remapping.c

598

iommu->name);

drivers/iommu/intel/irq_remapping.c

601

iommu_set_irq_remapping(iommu, eim_mode);

drivers/iommu/intel/irq_remapping.c

606

irq_domain_remove(iommu->ir_domain);

drivers/iommu/intel/irq_remapping.c

607

iommu->ir_domain = NULL;

drivers/iommu/intel/irq_remapping.c

617

iommu->ir_table = NULL;

drivers/iommu/intel/irq_remapping.c

622

static void intel_teardown_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

626

if (iommu && iommu->ir_table) {

drivers/iommu/intel/irq_remapping.c

627

if (iommu->ir_domain) {

drivers/iommu/intel/irq_remapping.c

628

fn = iommu->ir_domain->fwnode;

drivers/iommu/intel/irq_remapping.c

630

irq_domain_remove(iommu->ir_domain);

drivers/iommu/intel/irq_remapping.c

632

iommu->ir_domain = NULL;

drivers/iommu/intel/irq_remapping.c

634

iommu_free_pages(iommu->ir_table->base);

drivers/iommu/intel/irq_remapping.c

635

bitmap_free(iommu->ir_table->bitmap);

drivers/iommu/intel/irq_remapping.c

636

kfree(iommu->ir_table);

drivers/iommu/intel/irq_remapping.c

637

iommu->ir_table = NULL;

drivers/iommu/intel/irq_remapping.c

644

static void iommu_disable_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

649

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

656

if (!cap_esirtps(iommu->cap))

drivers/iommu/intel/irq_remapping.c

657

qi_global_iec(iommu);

drivers/iommu/intel/irq_remapping.c

659

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

661

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/irq_remapping.c

665

iommu->gcmd &= ~DMA_GCMD_IRE;

drivers/iommu/intel/irq_remapping.c

666

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

668

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

672

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

687

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

689

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

690

if (ecap_ir_support(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

691

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

692

intel_teardown_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

703

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

728

for_each_iommu(iommu, drhd)

drivers/iommu/intel/irq_remapping.c

729

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

741

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

742

if (eim && !ecap_eim_support(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

743

pr_info("%s does not support EIM\n", iommu->name);

drivers/iommu/intel/irq_remapping.c

753

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

754

if (intel_setup_irq_remapping(iommu)) {

drivers/iommu/intel/irq_remapping.c

756

iommu->name);

drivers/iommu/intel/irq_remapping.c

774

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

788

for_each_iommu(iommu, drhd)

drivers/iommu/intel/irq_remapping.c

789

if (!cap_pi_support(iommu->cap)) {

drivers/iommu/intel/irq_remapping.c

800

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

806

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

807

if (!ir_pre_enabled(iommu))

drivers/iommu/intel/irq_remapping.c

808

iommu_enable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

81

static void iommu_disable_irq_remapping(struct intel_iommu *iommu);

drivers/iommu/intel/irq_remapping.c

829

struct intel_iommu *iommu,

drivers/iommu/intel/irq_remapping.c

85

static bool ir_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

852

if (ir_hpet[count].iommu == iommu &&

drivers/iommu/intel/irq_remapping.c

855

else if (ir_hpet[count].iommu == NULL && free == -1)

drivers/iommu/intel/irq_remapping.c

863

ir_hpet[free].iommu = iommu;

drivers/iommu/intel/irq_remapping.c

87

return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);

drivers/iommu/intel/irq_remapping.c

874

struct intel_iommu *iommu,

drivers/iommu/intel/irq_remapping.c

897

if (ir_ioapic[count].iommu == iommu &&

drivers/iommu/intel/irq_remapping.c

90

static void clear_ir_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

900

else if (ir_ioapic[count].iommu == NULL && free == -1)

drivers/iommu/intel/irq_remapping.c

910

ir_ioapic[free].iommu = iommu;

drivers/iommu/intel/irq_remapping.c

913

scope->enumeration_id, drhd->address, iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

919

struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

92

iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;

drivers/iommu/intel/irq_remapping.c

933

ret = ir_parse_one_ioapic_scope(scope, iommu, drhd);

drivers/iommu/intel/irq_remapping.c

935

ret = ir_parse_one_hpet_scope(scope, iommu, drhd);

drivers/iommu/intel/irq_remapping.c

942

static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

947

if (ir_hpet[i].iommu == iommu)

drivers/iommu/intel/irq_remapping.c

948

ir_hpet[i].iommu = NULL;

drivers/iommu/intel/irq_remapping.c

95

static void init_ir_status(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

951

if (ir_ioapic[i].iommu == iommu)

drivers/iommu/intel/irq_remapping.c

952

ir_ioapic[i].iommu = NULL;

drivers/iommu/intel/irq_remapping.c

962

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

966

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

969

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

972

ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu);

drivers/iommu/intel/irq_remapping.c

99

gsts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/nested.c

134

static int domain_setup_nested(struct intel_iommu *iommu,

drivers/iommu/intel/nested.c

140

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/nested.c

142

return intel_pasid_setup_nested(iommu, dev, pasid, domain);

drivers/iommu/intel/nested.c

151

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/nested.c

155

if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))

drivers/iommu/intel/nested.c

158

if (context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/nested.c

173

ret = domain_setup_nested(iommu, dmar_domain, dev, pasid, old);

drivers/iommu/intel/nested.c

202

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/nested.c

207

if (!nested_supported(iommu) || flags & ~IOMMU_HWPT_ALLOC_PASID)

drivers/iommu/intel/nested.c

26

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/nested.c

43

ret = domain_attach_iommu(dmar_domain, iommu);

drivers/iommu/intel/nested.c

57

ret = intel_pasid_setup_nested(iommu, dev,

drivers/iommu/intel/nested.c

74

domain_detach_iommu(dmar_domain, iommu);

drivers/iommu/intel/pasid.c

151

entries = iommu_alloc_pages_node_sz(info->iommu->node,

drivers/iommu/intel/pasid.c

156

if (!ecap_coherent(info->iommu->ecap))

drivers/iommu/intel/pasid.c

171

if (!ecap_coherent(info->iommu->ecap))

drivers/iommu/intel/pasid.c

197

pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

208

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/pasid.c

212

devtlb_invalidation_with_pasid(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

236

qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);

drivers/iommu/intel/pasid.c

238

qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);

drivers/iommu/intel/pasid.c

241

void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.c

247

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

250

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

257

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

268

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

277

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

279

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

282

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

285

qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);

drivers/iommu/intel/pasid.c

287

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c

289

devtlb_invalidation_with_pasid(iommu, dev, pasid);

drivers/iommu/intel/pasid.c

291

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

302

static void pasid_flush_caches(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

306

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

309

if (cap_caching_mode(iommu->cap)) {

drivers/iommu/intel/pasid.c

310

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

311

qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);

drivers/iommu/intel/pasid.c

313

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/pasid.c

325

static void intel_pasid_flush_present(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

330

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

344

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

345

qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);

drivers/iommu/intel/pasid.c

347

devtlb_invalidation_with_pasid(iommu, dev, pasid);

drivers/iommu/intel/pasid.c

354

static void pasid_pte_config_first_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

359

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

373

pasid_set_address_width(pte, iommu->agaw);

drivers/iommu/intel/pasid.c

381

int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.c

387

if (!ecap_flts(iommu->ecap)) {

drivers/iommu/intel/pasid.c

389

iommu->name);

drivers/iommu/intel/pasid.c

393

if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/pasid.c

395

iommu->name);

drivers/iommu/intel/pasid.c

399

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

402

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

407

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

411

pasid_pte_config_first_level(iommu, pte, fsptptr, did, flags);

drivers/iommu/intel/pasid.c

413

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

415

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

423

static void pasid_pte_config_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

429

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

446

int intel_pasid_setup_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

458

if (!ecap_slts(iommu->ecap)) {

drivers/iommu/intel/pasid.c

460

iommu->name);

drivers/iommu/intel/pasid.c

464

did = domain_id_iommu(domain, iommu);

drivers/iommu/intel/pasid.c

466

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

469

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

474

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

478

pasid_pte_config_second_level(iommu, pte, domain, did);

drivers/iommu/intel/pasid.c

479

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

481

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

489

int intel_pasid_setup_dirty_tracking(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

496

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

500

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

510

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

519

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

527

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

529

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

546

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

548

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c

550

devtlb_invalidation_with_pasid(iommu, dev, pasid);

drivers/iommu/intel/pasid.c

558

static void pasid_pte_config_pass_through(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

561

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

565

pasid_set_address_width(pte, iommu->agaw);

drivers/iommu/intel/pasid.c

568

pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));

drivers/iommu/intel/pasid.c

572

int intel_pasid_setup_pass_through(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

578

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

581

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

586

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

590

pasid_pte_config_pass_through(iommu, pte, did);

drivers/iommu/intel/pasid.c

591

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

593

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

601

void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

607

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

610

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

616

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

618

intel_pasid_flush_present(iommu, dev, pasid, did, pte);

drivers/iommu/intel/pasid.c

621

static void pasid_pte_config_nestd(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

629

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

63

dir = iommu_alloc_pages_node_sz(info->iommu->node, GFP_KERNEL,

drivers/iommu/intel/pasid.c

675

int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.c

680

u16 did = domain_id_iommu(domain, iommu);

drivers/iommu/intel/pasid.c

688

if (!cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/pasid.c

700

if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) {

drivers/iommu/intel/pasid.c

702

iommu->name);

drivers/iommu/intel/pasid.c

706

if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) {

drivers/iommu/intel/pasid.c

708

iommu->name);

drivers/iommu/intel/pasid.c

712

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

715

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

719

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

723

pasid_pte_config_nestd(iommu, pte, s1_cfg, s2_domain, did);

drivers/iommu/intel/pasid.c

724

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

726

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

739

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

74

if (!ecap_coherent(info->iommu->ecap))

drivers/iommu/intel/pasid.c

743

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

744

context = iommu_context_addr(iommu, bus, devfn, false);

drivers/iommu/intel/pasid.c

746

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

752

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

753

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

801

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

819

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

827

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

830

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

831

context = iommu_context_addr(iommu, bus, devfn, true);

drivers/iommu/intel/pasid.c

833

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

837

if (context_present(context) && !context_copied(iommu, bus, devfn)) {

drivers/iommu/intel/pasid.c

838

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

842

if (context_copied(iommu, bus, devfn)) {

drivers/iommu/intel/pasid.c

844

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

855

iommu->flush.flush_context(iommu, 0,

drivers/iommu/intel/pasid.c

859

qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);

drivers/iommu/intel/pasid.c

860

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

drivers/iommu/intel/pasid.c

861

devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID);

drivers/iommu/intel/pasid.c

864

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

871

clear_context_copied(iommu, bus, devfn);

drivers/iommu/intel/pasid.c

875

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

883

if (cap_caching_mode(iommu->cap)) {

drivers/iommu/intel/pasid.c

884

iommu->flush.flush_context(iommu, 0,

drivers/iommu/intel/pasid.c

888

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c

937

qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn),

drivers/iommu/intel/pasid.c

960

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

968

iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn),

drivers/iommu/intel/pasid.c

976

if (!sm_supported(iommu)) {

drivers/iommu/intel/pasid.c

977

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.h

306

int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.h

309

int intel_pasid_setup_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

312

int intel_pasid_setup_dirty_tracking(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

315

int intel_pasid_setup_pass_through(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

317

int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.h

319

void intel_pasid_tear_down_entry(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

322

void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,

drivers/iommu/intel/perf.c

116

void dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size)

drivers/iommu/intel/perf.c

118

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

130

if (!dmar_latency_enabled(iommu, i))

drivers/iommu/intel/perf.c

18

bool dmar_latency_enabled(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.c

20

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

25

int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.c

31

if (dmar_latency_enabled(iommu, type))

drivers/iommu/intel/perf.c

35

if (!iommu->perf_statistic) {

drivers/iommu/intel/perf.c

36

iommu->perf_statistic = kzalloc_objs(*lstat, DMAR_LATENCY_NUM,

drivers/iommu/intel/perf.c

38

if (!iommu->perf_statistic) {

drivers/iommu/intel/perf.c

44

lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

57

void dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.c

59

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

62

if (!dmar_latency_enabled(iommu, type))

drivers/iommu/intel/perf.c

70

void dmar_latency_update(struct intel_iommu *iommu, enum latency_type type, u64 latency)

drivers/iommu/intel/perf.c

72

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

76

if (!dmar_latency_enabled(iommu, type))

drivers/iommu/intel/perf.h

38

int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type);

drivers/iommu/intel/perf.h

39

void dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type);

drivers/iommu/intel/perf.h

40

bool dmar_latency_enabled(struct intel_iommu *iommu, enum latency_type type);

drivers/iommu/intel/perf.h

41

void dmar_latency_update(struct intel_iommu *iommu, enum latency_type type,

drivers/iommu/intel/perf.h

43

void dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size);

drivers/iommu/intel/perf.h

46

dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.h

52

dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.h

57

dmar_latency_enabled(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.h

63

dmar_latency_update(struct intel_iommu *iommu, enum latency_type type, u64 latency)

drivers/iommu/intel/perf.h

68

dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size)

drivers/iommu/intel/perfmon.c

327

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

356

ecmd_submit_sync(iommu, DMA_ECMD_ENABLE, hwc->idx, 0);

drivers/iommu/intel/perfmon.c

364

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

368

ecmd_submit_sync(iommu, DMA_ECMD_DISABLE, hwc->idx, 0);

drivers/iommu/intel/perfmon.c

476

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

478

ecmd_submit_sync(iommu, DMA_ECMD_UNFREEZE, 0, 0);

drivers/iommu/intel/perfmon.c

484

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

486

ecmd_submit_sync(iommu, DMA_ECMD_FREEZE, 0, 0);

drivers/iommu/intel/perfmon.c

519

struct intel_iommu *iommu = dev_id;

drivers/iommu/intel/perfmon.c

521

if (!dmar_readl(iommu->reg + DMAR_PERFINTRSTS_REG))

drivers/iommu/intel/perfmon.c

524

iommu_pmu_counter_overflow(iommu->pmu);

drivers/iommu/intel/perfmon.c

527

dmar_writel(iommu->reg + DMAR_PERFINTRSTS_REG, DMA_PERFINTRSTS_PIS);

drivers/iommu/intel/perfmon.c

532

static int __iommu_pmu_register(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

534

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

536

iommu_pmu->pmu.name = iommu->name;

drivers/iommu/intel/perfmon.c

556

get_perf_reg_address(struct intel_iommu *iommu, u32 offset)

drivers/iommu/intel/perfmon.c

558

u32 off = dmar_readl(iommu->reg + offset);

drivers/iommu/intel/perfmon.c

560

return iommu->reg + off;

drivers/iommu/intel/perfmon.c

563

int alloc_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

570

if (!ecap_pms(iommu->ecap))

drivers/iommu/intel/perfmon.c

574

if (!cap_ecmds(iommu->cap))

drivers/iommu/intel/perfmon.c

577

perfcap = dmar_readq(iommu->reg + DMAR_PERFCAP_REG);

drivers/iommu/intel/perfmon.c

591

if (!ecmd_has_pmu_essential(iommu))

drivers/iommu/intel/perfmon.c

620

pcap = dmar_readq(iommu->reg + DMAR_PERFEVNTCAP_REG +

drivers/iommu/intel/perfmon.c

644

iommu_pmu->cfg_reg = get_perf_reg_address(iommu, DMAR_PERFCFGOFF_REG);

drivers/iommu/intel/perfmon.c

645

iommu_pmu->cntr_reg = get_perf_reg_address(iommu, DMAR_PERFCNTROFF_REG);

drivers/iommu/intel/perfmon.c

646

iommu_pmu->overflow = get_perf_reg_address(iommu, DMAR_PERFOVFOFF_REG);

drivers/iommu/intel/perfmon.c

690

iommu_pmu->iommu = iommu;

drivers/iommu/intel/perfmon.c

691

iommu->pmu = iommu_pmu;

drivers/iommu/intel/perfmon.c

707

void free_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

709

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

723

iommu->pmu = NULL;

drivers/iommu/intel/perfmon.c

726

static int iommu_pmu_set_interrupt(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

728

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

731

irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PERF + iommu->seq_id, iommu->node, iommu);

drivers/iommu/intel/perfmon.c

735

snprintf(iommu_pmu->irq_name, sizeof(iommu_pmu->irq_name), "dmar%d-perf", iommu->seq_id);

drivers/iommu/intel/perfmon.c

737

iommu->perf_irq = irq;

drivers/iommu/intel/perfmon.c

739

IRQF_ONESHOT, iommu_pmu->irq_name, iommu);

drivers/iommu/intel/perfmon.c

742

iommu->perf_irq = 0;

drivers/iommu/intel/perfmon.c

748

static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

750

if (!iommu->perf_irq)

drivers/iommu/intel/perfmon.c

753

free_irq(iommu->perf_irq, iommu);

drivers/iommu/intel/perfmon.c

754

dmar_free_hwirq(iommu->perf_irq);

drivers/iommu/intel/perfmon.c

755

iommu->perf_irq = 0;

drivers/iommu/intel/perfmon.c

758

void iommu_pmu_register(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

760

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

765

if (__iommu_pmu_register(iommu))

drivers/iommu/intel/perfmon.c

769

if (iommu_pmu_set_interrupt(iommu))

drivers/iommu/intel/perfmon.c

777

pr_err("Failed to register PMU for iommu (seq_id = %d)\n", iommu->seq_id);

drivers/iommu/intel/perfmon.c

778

free_iommu_pmu(iommu);

drivers/iommu/intel/perfmon.c

781

void iommu_pmu_unregister(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

783

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

788

iommu_pmu_unset_interrupt(iommu);

drivers/iommu/intel/perfmon.h

39

int alloc_iommu_pmu(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

40

void free_iommu_pmu(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

41

void iommu_pmu_register(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

42

void iommu_pmu_unregister(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

45

alloc_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.h

51

free_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.h

56

iommu_pmu_register(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.h

61

iommu_pmu_unregister(struct intel_iommu *iommu)

drivers/iommu/intel/prq.c

112

qi_desc_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH, &desc[1]);

drivers/iommu/intel/prq.c

121

reinit_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

122

qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);

drivers/iommu/intel/prq.c

123

if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {

drivers/iommu/intel/prq.c

124

wait_for_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

137

static void handle_bad_prq_event(struct intel_iommu *iommu,

drivers/iommu/intel/prq.c

143

iommu->name, ((unsigned long long *)req)[0],

drivers/iommu/intel/prq.c

156

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/prq.c

175

static void intel_prq_report(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/prq.c

199

struct intel_iommu *iommu = d;

drivers/iommu/intel/prq.c

209

writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);

drivers/iommu/intel/prq.c

211

tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

212

head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

215

req = &iommu->prq[head / sizeof(*req)];

drivers/iommu/intel/prq.c

220

iommu->name);

drivers/iommu/intel/prq.c

222

handle_bad_prq_event(iommu, req, QI_RESP_INVALID);

drivers/iommu/intel/prq.c

228

iommu->name);

drivers/iommu/intel/prq.c

234

iommu->name);

drivers/iommu/intel/prq.c

246

mutex_lock(&iommu->iopf_lock);

drivers/iommu/intel/prq.c

247

dev = device_rbtree_find(iommu, req->rid);

drivers/iommu/intel/prq.c

249

mutex_unlock(&iommu->iopf_lock);

drivers/iommu/intel/prq.c

253

intel_prq_report(iommu, dev, req);

drivers/iommu/intel/prq.c

254

trace_prq_report(iommu, dev, req->qw_0, req->qw_1,

drivers/iommu/intel/prq.c

256

iommu->prq_seq_number++);

drivers/iommu/intel/prq.c

257

mutex_unlock(&iommu->iopf_lock);

drivers/iommu/intel/prq.c

262

dmar_writeq(iommu->reg + DMAR_PQH_REG, tail);

drivers/iommu/intel/prq.c

268

if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {

drivers/iommu/intel/prq.c

270

iommu->name);

drivers/iommu/intel/prq.c

271

head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

272

tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

274

iopf_queue_discard_partial(iommu->iopf_queue);

drivers/iommu/intel/prq.c

275

writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG);

drivers/iommu/intel/prq.c

277

iommu->name);

drivers/iommu/intel/prq.c

281

if (!completion_done(&iommu->prq_complete))

drivers/iommu/intel/prq.c

282

complete(&iommu->prq_complete);

drivers/iommu/intel/prq.c

287

int intel_iommu_enable_prq(struct intel_iommu *iommu)

drivers/iommu/intel/prq.c

292

iommu->prq =

drivers/iommu/intel/prq.c

293

iommu_alloc_pages_node_sz(iommu->node, GFP_KERNEL, PRQ_SIZE);

drivers/iommu/intel/prq.c

294

if (!iommu->prq) {

drivers/iommu/intel/prq.c

296

iommu->name);

drivers/iommu/intel/prq.c

300

irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu);

drivers/iommu/intel/prq.c

303

iommu->name);

drivers/iommu/intel/prq.c

307

iommu->pr_irq = irq;

drivers/iommu/intel/prq.c

309

snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name),

drivers/iommu/intel/prq.c

310

"dmar%d-iopfq", iommu->seq_id);

drivers/iommu/intel/prq.c

311

iopfq = iopf_queue_alloc(iommu->iopfq_name);

drivers/iommu/intel/prq.c

313

pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu->name);

drivers/iommu/intel/prq.c

317

iommu->iopf_queue = iopfq;

drivers/iommu/intel/prq.c

319

snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id);

drivers/iommu/intel/prq.c

322

iommu->prq_name, iommu);

drivers/iommu/intel/prq.c

325

iommu->name);

drivers/iommu/intel/prq.c

328

dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);

drivers/iommu/intel/prq.c

329

dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);

drivers/iommu/intel/prq.c

330

dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER);

drivers/iommu/intel/prq.c

332

init_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

337

iopf_queue_free(iommu->iopf_queue);

drivers/iommu/intel/prq.c

338

iommu->iopf_queue = NULL;

drivers/iommu/intel/prq.c

341

iommu->pr_irq = 0;

drivers/iommu/intel/prq.c

343

iommu_free_pages(iommu->prq);

drivers/iommu/intel/prq.c

344

iommu->prq = NULL;

drivers/iommu/intel/prq.c

349

int intel_iommu_finish_prq(struct intel_iommu *iommu)

drivers/iommu/intel/prq.c

351

dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);

drivers/iommu/intel/prq.c

352

dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);

drivers/iommu/intel/prq.c

353

dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL);

drivers/iommu/intel/prq.c

355

if (iommu->pr_irq) {

drivers/iommu/intel/prq.c

356

free_irq(iommu->pr_irq, iommu);

drivers/iommu/intel/prq.c

357

dmar_free_hwirq(iommu->pr_irq);

drivers/iommu/intel/prq.c

358

iommu->pr_irq = 0;

drivers/iommu/intel/prq.c

361

if (iommu->iopf_queue) {

drivers/iommu/intel/prq.c

362

iopf_queue_free(iommu->iopf_queue);

drivers/iommu/intel/prq.c

363

iommu->iopf_queue = NULL;

drivers/iommu/intel/prq.c

366

iommu_free_pages(iommu->prq);

drivers/iommu/intel/prq.c

367

iommu->prq = NULL;

drivers/iommu/intel/prq.c

376

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/prq.c

395

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/prq.c

64

struct intel_iommu *iommu;

drivers/iommu/intel/prq.c

73

iommu = info->iommu;

drivers/iommu/intel/prq.c

76

did = domain ? domain_id_iommu(domain, iommu) : FLPT_DEFAULT_DID;

drivers/iommu/intel/prq.c

83

reinit_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

84

tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

85

head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

89

req = &iommu->prq[head / sizeof(*req)];

drivers/iommu/intel/prq.c

97

wait_for_completion(&iommu->prq_complete);

drivers/iommu/intel/svm.c

116

struct intel_iommu *iommu;

drivers/iommu/intel/svm.c

121

iommu = info->iommu;

drivers/iommu/intel/svm.c

122

if (!iommu)

drivers/iommu/intel/svm.c

125

if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE))

drivers/iommu/intel/svm.c

153

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/svm.c

177

ret = __domain_setup_first_level(iommu, dev, pasid,

drivers/iommu/intel/svm.c

28

void intel_svm_check(struct intel_iommu *iommu)

drivers/iommu/intel/svm.c

30

if (!pasid_supported(iommu))

drivers/iommu/intel/svm.c

34

!cap_fl1gp_support(iommu->cap)) {

drivers/iommu/intel/svm.c

36

iommu->name);

drivers/iommu/intel/svm.c

41

!cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/svm.c

43

iommu->name);

drivers/iommu/intel/svm.c

47

iommu->flags |= VTD_FLAG_SVM_CAPABLE;

drivers/iommu/intel/svm.c

92

intel_pasid_tear_down_entry(info->iommu, dev_pasid->dev,

drivers/iommu/intel/trace.h

105

__assign_str(iommu);

drivers/iommu/intel/trace.h

113

__get_str(iommu), __get_str(dev),

drivers/iommu/intel/trace.h

138

__string(iommu, tag->iommu->name)

drivers/iommu/intel/trace.h

150

__assign_str(iommu);

drivers/iommu/intel/trace.h

162

__get_str(iommu), __get_str(dev), __entry->pasid,

drivers/iommu/intel/trace.h

189

#define TRACE_INCLUDE_PATH ../../drivers/iommu/intel/

drivers/iommu/intel/trace.h

22

TP_PROTO(struct intel_iommu *iommu, u64 qw0, u64 qw1, u64 qw2, u64 qw3),

drivers/iommu/intel/trace.h

24

TP_ARGS(iommu, qw0, qw1, qw2, qw3),

drivers/iommu/intel/trace.h

31

__string(iommu, iommu->name)

drivers/iommu/intel/trace.h

35

__assign_str(iommu);

drivers/iommu/intel/trace.h

53

__get_str(iommu),

drivers/iommu/intel/trace.h

59

TP_PROTO(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/trace.h

63

TP_ARGS(iommu, dev, dw0, dw1, dw2, dw3, seq),

drivers/iommu/intel/trace.h

71

__string(iommu, iommu->name)

drivers/iommu/intel/trace.h

82

__assign_str(iommu);

drivers/iommu/intel/trace.h

87

__get_str(iommu), __get_str(dev), __entry->seq,

drivers/iommu/intel/trace.h

97

__string(iommu, tag->iommu->name)

drivers/iommu/io-pgfault.c

23

struct dev_iommu *param = dev->iommu;

drivers/iommu/io-pgfault.c

307

iopf_param = rcu_dereference_check(dev->iommu->fault_param, true);

drivers/iommu/io-pgfault.c

388

struct dev_iommu *param = dev->iommu;

drivers/iommu/io-pgfault.c

455

struct dev_iommu *param = dev->iommu;

drivers/iommu/iommu-priv.h

19

return dev->iommu->iommu_dev->ops;

drivers/iommu/iommu-priv.h

33

int iommu_device_register_bus(struct iommu_device *iommu,

drivers/iommu/iommu-priv.h

37

void iommu_device_unregister_bus(struct iommu_device *iommu,

drivers/iommu/iommu-priv.h

41

int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu);

drivers/iommu/iommu-sva.c

33

if (iommu_mm->pasid >= dev->iommu->max_pasids)

drivers/iommu/iommu-sysfs.c

106

int iommu_device_link(struct iommu_device *iommu, struct device *link)

drivers/iommu/iommu-sysfs.c

110

ret = sysfs_add_link_to_group(&iommu->dev->kobj, "devices",

drivers/iommu/iommu-sysfs.c

115

ret = sysfs_create_link_nowarn(&link->kobj, &iommu->dev->kobj, "iommu");

drivers/iommu/iommu-sysfs.c

117

sysfs_remove_link_from_group(&iommu->dev->kobj, "devices",

drivers/iommu/iommu-sysfs.c

123

void iommu_device_unlink(struct iommu_device *iommu, struct device *link)

drivers/iommu/iommu-sysfs.c

126

sysfs_remove_link_from_group(&iommu->dev->kobj, "devices", dev_name(link));

drivers/iommu/iommu-sysfs.c

54

int iommu_device_sysfs_add(struct iommu_device *iommu,

drivers/iommu/iommu-sysfs.c

62

iommu->dev = kzalloc_obj(*iommu->dev);

drivers/iommu/iommu-sysfs.c

63

if (!iommu->dev)

drivers/iommu/iommu-sysfs.c

66

device_initialize(iommu->dev);

drivers/iommu/iommu-sysfs.c

68

iommu->dev->class = &iommu_class;

drivers/iommu/iommu-sysfs.c

69

iommu->dev->parent = parent;

drivers/iommu/iommu-sysfs.c

70

iommu->dev->groups = groups;

drivers/iommu/iommu-sysfs.c

73

ret = kobject_set_name_vargs(&iommu->dev->kobj, fmt, vargs);

drivers/iommu/iommu-sysfs.c

78

ret = device_add(iommu->dev);

drivers/iommu/iommu-sysfs.c

82

dev_set_drvdata(iommu->dev, iommu);

drivers/iommu/iommu-sysfs.c

87

put_device(iommu->dev);

drivers/iommu/iommu-sysfs.c

92

void iommu_device_sysfs_remove(struct iommu_device *iommu)

drivers/iommu/iommu-sysfs.c

94

dev_set_drvdata(iommu->dev, NULL);

drivers/iommu/iommu-sysfs.c

95

device_unregister(iommu->dev);

drivers/iommu/iommu-sysfs.c

96

iommu->dev = NULL;

drivers/iommu/iommu.c

1200

dev->iommu->require_direct = 1;

drivers/iommu/iommu.c

1584

struct iommu_device *iommu = dev->iommu->iommu_dev;

drivers/iommu/iommu.c

1586

if (!iommu->singleton_group) {

drivers/iommu/iommu.c

1592

iommu->singleton_group = group;

drivers/iommu/iommu.c

1594

return iommu_group_ref_get(iommu->singleton_group);

drivers/iommu/iommu.c

1714

dev->iommu->max_pasids ? IOMMU_HWPT_ALLOC_PASID : 0);

drivers/iommu/iommu.c

2153

dev->iommu->attach_deferred = 0;

drivers/iommu/iommu.c

2203

if (!dev->iommu || !dev->iommu->attach_deferred)

drivers/iommu/iommu.c

2379

if (dev->iommu->require_direct &&

drivers/iommu/iommu.c

2387

if (dev->iommu->attach_deferred) {

drivers/iommu/iommu.c

2390

dev->iommu->attach_deferred = 0;

drivers/iommu/iommu.c

253

if (dev->iommu && dev->iommu->iommu_dev == data)

drivers/iommu/iommu.c

267

int iommu_device_register(struct iommu_device *iommu,

drivers/iommu/iommu.c

276

iommu->ops = ops;

drivers/iommu/iommu.c

278

iommu->fwnode = dev_fwnode(hwdev);

drivers/iommu/iommu.c

281

list_add_tail(&iommu->list, &iommu_device_list);

drivers/iommu/iommu.c

287

iommu_device_unregister(iommu);

drivers/iommu/iommu.c

289

WRITE_ONCE(iommu->ready, true);

drivers/iommu/iommu.c

294

void iommu_device_unregister(struct iommu_device *iommu)

drivers/iommu/iommu.c

297

bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group);

drivers/iommu/iommu.c

2981

const struct iommu_device *iommu, *ret = NULL;

drivers/iommu/iommu.c

2984

list_for_each_entry(iommu, &iommu_device_list, list)

drivers/iommu/iommu.c

2985

if (iommu->fwnode == fwnode) {

drivers/iommu/iommu.c

2986

ret = iommu;

drivers/iommu/iommu.c

2995

const struct iommu_device *iommu = iommu_from_fwnode(fwnode);

drivers/iommu/iommu.c

2997

return iommu ? iommu->ops : NULL;

drivers/iommu/iommu.c

300

list_del(&iommu->list);

drivers/iommu/iommu.c

3002

const struct iommu_device *iommu = iommu_from_fwnode(iommu_fwnode);

drivers/iommu/iommu.c

3005

if (!iommu)

drivers/iommu/iommu.c

3007

if (!dev->iommu && !READ_ONCE(iommu->ready))

drivers/iommu/iommu.c

3011

return iommu->ops == iommu_fwspec_ops(fwspec) ? 0 : -EINVAL;

drivers/iommu/iommu.c

304

iommu_group_put(iommu->singleton_group);

drivers/iommu/iommu.c

305

iommu->singleton_group = NULL;

drivers/iommu/iommu.c

310

void iommu_device_unregister_bus(struct iommu_device *iommu,

drivers/iommu/iommu.c

3116

gdev->dev->iommu->iommu_dev->dev,

drivers/iommu/iommu.c

315

fwnode_remove_software_node(iommu->fwnode);

drivers/iommu/iommu.c

316

iommu_device_unregister(iommu);

drivers/iommu/iommu.c

325

int iommu_device_register_bus(struct iommu_device *iommu,

drivers/iommu/iommu.c

332

iommu->ops = ops;

drivers/iommu/iommu.c

338

iommu->fwnode = fwnode_create_software_node(NULL, NULL);

drivers/iommu/iommu.c

339

if (IS_ERR(iommu->fwnode)) {

drivers/iommu/iommu.c

341

return PTR_ERR(iommu->fwnode);

drivers/iommu/iommu.c

345

list_add_tail(&iommu->list, &iommu_device_list);

drivers/iommu/iommu.c

3491

if (device->dev->iommu->max_pasids > 0) {

drivers/iommu/iommu.c

350

iommu_device_unregister_bus(iommu, bus, nb);

drivers/iommu/iommu.c

3506

if (device->dev->iommu->max_pasids > 0) {

drivers/iommu/iommu.c

3529

if (device->dev->iommu->max_pasids > 0)

drivers/iommu/iommu.c

353

WRITE_ONCE(iommu->ready, true);

drivers/iommu/iommu.c

358

int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu)

drivers/iommu/iommu.c

3588

if ((device->dev->iommu->max_pasids > 0) &&

drivers/iommu/iommu.c

3589

(pasid >= device->dev->iommu->max_pasids)) {

drivers/iommu/iommu.c

363

rc = iommu_fwspec_init(dev, iommu->fwnode);

drivers/iommu/iommu.c

3754

if (!dev->iommu->max_pasids)

drivers/iommu/iommu.c

3762

dev->iommu->max_pasids - 1, GFP_KERNEL);

drivers/iommu/iommu.c

379

struct dev_iommu *param = dev->iommu;

drivers/iommu/iommu.c

391

dev->iommu = param;

drivers/iommu/iommu.c

397

struct dev_iommu *param = dev->iommu;

drivers/iommu/iommu.c

399

dev->iommu = NULL;

drivers/iommu/iommu.c

413

return dev->iommu && dev->iommu->iommu_dev;

drivers/iommu/iommu.c

431

return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);

drivers/iommu/iommu.c

439

dev->iommu->priv = priv;

drivers/iommu/iommu.c

463

if (!dev->iommu->fwspec && !dev->driver && dev->bus->dma_configure) {

drivers/iommu/iommu.c

468

if (!dev->iommu || dev->iommu_group)

drivers/iommu/iommu.c

478

ops = iommu_fwspec_ops(dev->iommu->fwspec);

drivers/iommu/iommu.c

494

dev->iommu->iommu_dev = iommu_dev;

drivers/iommu/iommu.c

509

dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);

drivers/iommu/iommu.c

511

dev->iommu->attach_deferred = ops->is_attach_deferred(dev);

drivers/iommu/iommu.c

522

dev->iommu->iommu_dev = NULL;

drivers/iommu/iommu.c

534

iommu_device_unlink(dev->iommu->iommu_dev, dev);

drivers/iommu/iommu.c

553

if (!dev->iommu->attach_deferred && ops->release_domain) {

drivers/iommu/iommu.c

561

if (dev->iommu->require_direct && ops->identity_domain &&

drivers/iommu/iommu.c

765

if (dev->iommu)

drivers/iommu/iommufd/device.c

1636

if (idev->dev->iommu->max_pasids) {

drivers/iommu/iommufd/device.c

1637

cmd->out_max_pasid_log2 = ilog2(idev->dev->iommu->max_pasids);

drivers/iommu/iommufd/selftest.c

118

struct pt_iommu iommu;

drivers/iommu/iommufd/selftest.c

123

PT_IOMMU_CHECK_DOMAIN(struct mock_iommu_domain, iommu, domain);

drivers/iommu/iommufd/selftest.c

124

PT_IOMMU_CHECK_DOMAIN(struct mock_iommu_domain, amdv1.iommu, domain);

drivers/iommu/iommufd/selftest.c

1734

if (!(mock->flags & MOCK_DIRTY_TRACK) || !mock->iommu.ops->set_dirty) {

drivers/iommu/iommufd/selftest.c

1755

mock->iommu.ops->set_dirty(&mock->iommu, iova + i * page_size);

drivers/iommu/iommufd/selftest.c

392

pt_iommu_deinit(&mock->iommu);

drivers/iommu/iommufd/selftest.c

449

mock->amdv1.iommu.nid = NUMA_NO_NODE;

drivers/iommu/ipmmu-vmsa.c

1095

ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, "%s",

drivers/iommu/ipmmu-vmsa.c

1100

ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev);

drivers/iommu/ipmmu-vmsa.c

1102

iommu_device_sysfs_remove(&mmu->iommu);

drivers/iommu/ipmmu-vmsa.c

1111

iommu_device_sysfs_remove(&mmu->iommu);

drivers/iommu/ipmmu-vmsa.c

1112

iommu_device_unregister(&mmu->iommu);

drivers/iommu/ipmmu-vmsa.c

58

struct iommu_device iommu;

drivers/iommu/ipmmu-vmsa.c

846

return &mmu->iommu;

drivers/iommu/msm_iommu.c

120

struct msm_iommu_dev *iommu = NULL;

drivers/iommu/msm_iommu.c

124

list_for_each_entry(iommu, &priv->list_attached, dom_node) {

drivers/iommu/msm_iommu.c

125

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

129

list_for_each_entry(master, &iommu->ctx_list, list)

drivers/iommu/msm_iommu.c

130

SET_CTX_TLBIALL(iommu->base, master->num, 0);

drivers/iommu/msm_iommu.c

132

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

142

struct msm_iommu_dev *iommu = NULL;

drivers/iommu/msm_iommu.c

147

list_for_each_entry(iommu, &priv->list_attached, dom_node) {

drivers/iommu/msm_iommu.c

148

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

152

list_for_each_entry(master, &iommu->ctx_list, list) {

drivers/iommu/msm_iommu.c

156

iova |= GET_CONTEXTIDR_ASID(iommu->base,

drivers/iommu/msm_iommu.c

158

SET_TLBIVA(iommu->base, master->num, iova);

drivers/iommu/msm_iommu.c

163

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

206

static void config_mids(struct msm_iommu_dev *iommu,

drivers/iommu/msm_iommu.c

215

SET_M2VCBR_N(iommu->base, mid, 0);

drivers/iommu/msm_iommu.c

216

SET_CBACR_N(iommu->base, ctx, 0);

drivers/iommu/msm_iommu.c

219

SET_VMID(iommu->base, mid, 0);

drivers/iommu/msm_iommu.c

222

SET_CBNDX(iommu->base, mid, ctx);

drivers/iommu/msm_iommu.c

225

SET_CBVMID(iommu->base, ctx, 0);

drivers/iommu/msm_iommu.c

228

SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx);

drivers/iommu/msm_iommu.c

231

SET_NSCFG(iommu->base, mid, 3);

drivers/iommu/msm_iommu.c

363

struct msm_iommu_dev *iommu, *ret = NULL;

drivers/iommu/msm_iommu.c

366

list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {

drivers/iommu/msm_iommu.c

367

master = list_first_entry(&iommu->ctx_list,

drivers/iommu/msm_iommu.c

371

ret = iommu;

drivers/iommu/msm_iommu.c

381

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

385

iommu = find_iommu_for_dev(dev);

drivers/iommu/msm_iommu.c

388

if (!iommu)

drivers/iommu/msm_iommu.c

391

return &iommu->iommu;

drivers/iommu/msm_iommu.c

399

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

407

list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {

drivers/iommu/msm_iommu.c

408

master = list_first_entry(&iommu->ctx_list,

drivers/iommu/msm_iommu.c

412

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

416

list_for_each_entry(master, &iommu->ctx_list, list) {

drivers/iommu/msm_iommu.c

423

msm_iommu_alloc_ctx(iommu->context_map,

drivers/iommu/msm_iommu.c

424

0, iommu->ncb);

drivers/iommu/msm_iommu.c

429

config_mids(iommu, master);

drivers/iommu/msm_iommu.c

430

__program_context(iommu->base, master->num,

drivers/iommu/msm_iommu.c

433

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

434

list_add(&iommu->dom_node, &priv->list_attached);

drivers/iommu/msm_iommu.c

450

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

461

list_for_each_entry(iommu, &priv->list_attached, dom_node) {

drivers/iommu/msm_iommu.c

462

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

466

list_for_each_entry(master, &iommu->ctx_list, list) {

drivers/iommu/msm_iommu.c

467

msm_iommu_free_ctx(iommu->context_map, master->num);

drivers/iommu/msm_iommu.c

468

__reset_context(iommu->base, master->num);

drivers/iommu/msm_iommu.c

470

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

530

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

539

iommu = list_first_entry(&priv->list_attached,

drivers/iommu/msm_iommu.c

54

static int __enable_clocks(struct msm_iommu_dev *iommu)

drivers/iommu/msm_iommu.c

542

if (list_empty(&iommu->ctx_list))

drivers/iommu/msm_iommu.c

545

master = list_first_entry(&iommu->ctx_list,

drivers/iommu/msm_iommu.c

550

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

555

SET_CTX_TLBIALL(iommu->base, master->num, 0);

drivers/iommu/msm_iommu.c

556

SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA);

drivers/iommu/msm_iommu.c

558

par = GET_PAR(iommu->base, master->num);

drivers/iommu/msm_iommu.c

561

if (GET_NOFAULT_SS(iommu->base, master->num))

drivers/iommu/msm_iommu.c

566

if (GET_FAULT(iommu->base, master->num))

drivers/iommu/msm_iommu.c

569

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

58

ret = clk_enable(iommu->pclk);

drivers/iommu/msm_iommu.c

601

struct msm_iommu_dev **iommu,

drivers/iommu/msm_iommu.c

607

if (list_empty(&(*iommu)->ctx_list)) {

drivers/iommu/msm_iommu.c

614

list_add(&master->list, &(*iommu)->ctx_list);

drivers/iommu/msm_iommu.c

62

if (iommu->clk) {

drivers/iommu/msm_iommu.c

63

ret = clk_enable(iommu->clk);

drivers/iommu/msm_iommu.c

632

struct msm_iommu_dev *iommu = NULL, *iter;

drivers/iommu/msm_iommu.c

639

iommu = iter;

drivers/iommu/msm_iommu.c

644

if (!iommu) {

drivers/iommu/msm_iommu.c

649

ret = insert_iommu_master(dev, &iommu, spec);

drivers/iommu/msm_iommu.c

65

clk_disable(iommu->pclk);

drivers/iommu/msm_iommu.c

658

struct msm_iommu_dev *iommu = dev_id;

drivers/iommu/msm_iommu.c

664

if (!iommu) {

drivers/iommu/msm_iommu.c

670

pr_err("base = %08x\n", (unsigned int)iommu->base);

drivers/iommu/msm_iommu.c

672

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

676

for (i = 0; i < iommu->ncb; i++) {

drivers/iommu/msm_iommu.c

677

fsr = GET_FSR(iommu->base, i);

drivers/iommu/msm_iommu.c

681

print_ctx_regs(iommu->base, i);

drivers/iommu/msm_iommu.c

682

SET_FSR(iommu->base, i, 0x4000000F);

drivers/iommu/msm_iommu.c

685

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

71

static void __disable_clocks(struct msm_iommu_dev *iommu)

drivers/iommu/msm_iommu.c

718

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

721

iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/msm_iommu.c

722

if (!iommu)

drivers/iommu/msm_iommu.c

725

iommu->dev = &pdev->dev;

drivers/iommu/msm_iommu.c

726

INIT_LIST_HEAD(&iommu->ctx_list);

drivers/iommu/msm_iommu.c

728

iommu->pclk = devm_clk_get_prepared(iommu->dev, "smmu_pclk");

drivers/iommu/msm_iommu.c

729

if (IS_ERR(iommu->pclk))

drivers/iommu/msm_iommu.c

73

if (iommu->clk)

drivers/iommu/msm_iommu.c

730

return dev_err_probe(iommu->dev, PTR_ERR(iommu->pclk),

drivers/iommu/msm_iommu.c

733

iommu->clk = devm_clk_get_prepared(iommu->dev, "iommu_clk");

drivers/iommu/msm_iommu.c

734

if (IS_ERR(iommu->clk))

drivers/iommu/msm_iommu.c

735

return dev_err_probe(iommu->dev, PTR_ERR(iommu->clk),

drivers/iommu/msm_iommu.c

739

iommu->base = devm_ioremap_resource(iommu->dev, r);

drivers/iommu/msm_iommu.c

74

clk_disable(iommu->clk);

drivers/iommu/msm_iommu.c

740

if (IS_ERR(iommu->base)) {

drivers/iommu/msm_iommu.c

741

ret = dev_err_probe(iommu->dev, PTR_ERR(iommu->base), "could not get iommu base\n");

drivers/iommu/msm_iommu.c

746

iommu->irq = platform_get_irq(pdev, 0);

drivers/iommu/msm_iommu.c

747

if (iommu->irq < 0)

drivers/iommu/msm_iommu.c

75

clk_disable(iommu->pclk);

drivers/iommu/msm_iommu.c

750

ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val);

drivers/iommu/msm_iommu.c

752

dev_err(iommu->dev, "could not get ncb\n");

drivers/iommu/msm_iommu.c

755

iommu->ncb = val;

drivers/iommu/msm_iommu.c

757

msm_iommu_reset(iommu->base, iommu->ncb);

drivers/iommu/msm_iommu.c

758

SET_M(iommu->base, 0, 1);

drivers/iommu/msm_iommu.c

759

SET_PAR(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

760

SET_V2PCFG(iommu->base, 0, 1);

drivers/iommu/msm_iommu.c

761

SET_V2PPR(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

762

par = GET_PAR(iommu->base, 0);

drivers/iommu/msm_iommu.c

763

SET_V2PCFG(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

764

SET_M(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

771

ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL,

drivers/iommu/msm_iommu.c

775

iommu);

drivers/iommu/msm_iommu.c

777

pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret);

drivers/iommu/msm_iommu.c

781

list_add(&iommu->dev_node, &qcom_iommu_devices);

drivers/iommu/msm_iommu.c

783

ret = iommu_device_sysfs_add(&iommu->iommu, iommu->dev, NULL,

drivers/iommu/msm_iommu.c

790

ret = iommu_device_register(&iommu->iommu, &msm_iommu_ops, &pdev->dev);

drivers/iommu/msm_iommu.c

797

iommu->base, iommu->irq, iommu->ncb);

drivers/iommu/msm_iommu.h

60

struct iommu_device iommu;

drivers/iommu/mtk_iommu.c

1429

ret = iommu_device_sysfs_add(&data->iommu, dev, NULL,

drivers/iommu/mtk_iommu.c

1434

ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev);

drivers/iommu/mtk_iommu.c

1446

iommu_device_unregister(&data->iommu);

drivers/iommu/mtk_iommu.c

1448

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu.c

1468

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu.c

1469

iommu_device_unregister(&data->iommu);

drivers/iommu/mtk_iommu.c

264

struct iommu_device iommu;

drivers/iommu/mtk_iommu.c

886

return &data->iommu;

drivers/iommu/mtk_iommu.c

925

return &data->iommu;

drivers/iommu/mtk_iommu_v1.c

114

struct iommu_device iommu;

drivers/iommu/mtk_iommu_v1.c

514

return &data->iommu;

drivers/iommu/mtk_iommu_v1.c

691

ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL,

drivers/iommu/mtk_iommu_v1.c

696

ret = iommu_device_register(&data->iommu, &mtk_iommu_v1_ops, dev);

drivers/iommu/mtk_iommu_v1.c

706

iommu_device_unregister(&data->iommu);

drivers/iommu/mtk_iommu_v1.c

708

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu_v1.c

723

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu_v1.c

724

iommu_device_unregister(&data->iommu);

drivers/iommu/of_iommu.c

131

dev_iommu_present = dev->iommu;

drivers/iommu/of_iommu.c

154

else if (err && dev->iommu)

drivers/iommu/omap-iommu.c

1214

err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL,

drivers/iommu/omap-iommu.c

1222

err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);

drivers/iommu/omap-iommu.c

1236

iommu_device_sysfs_remove(&obj->iommu);

drivers/iommu/omap-iommu.c

1245

iommu_device_sysfs_remove(&obj->iommu);

drivers/iommu/omap-iommu.c

1247

iommu_device_unregister(&obj->iommu);

drivers/iommu/omap-iommu.c

1303

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1320

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1321

for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

1322

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1333

iommu--;

drivers/iommu/omap-iommu.c

1334

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1349

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1357

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1358

for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

1359

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1391

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1398

odomain->iommus = kzalloc_objs(*iommu, odomain->num_iommus, GFP_ATOMIC);

drivers/iommu/omap-iommu.c

1402

iommu = odomain->iommus;

drivers/iommu/omap-iommu.c

1403

for (i = 0; i < odomain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

1404

iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC);

drivers/iommu/omap-iommu.c

1405

if (!iommu->pgtable)

drivers/iommu/omap-iommu.c

1412

if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable,

drivers/iommu/omap-iommu.c

1423

struct omap_iommu_device *iommu = odomain->iommus;

drivers/iommu/omap-iommu.c

1425

for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++)

drivers/iommu/omap-iommu.c

1426

kfree(iommu->pgtable);

drivers/iommu/omap-iommu.c

1438

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1464

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1465

for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) {

drivers/iommu/omap-iommu.c

1468

ret = omap_iommu_attach(oiommu, iommu->pgtable);

drivers/iommu/omap-iommu.c

1475

iommu->iommu_dev = oiommu;

drivers/iommu/omap-iommu.c

1484

iommu--;

drivers/iommu/omap-iommu.c

1486

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1488

iommu->iommu_dev = NULL;

drivers/iommu/omap-iommu.c

1502

struct omap_iommu_device *iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1521

iommu += (omap_domain->num_iommus - 1);

drivers/iommu/omap-iommu.c

1523

for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) {

drivers/iommu/omap-iommu.c

1524

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1528

iommu->iommu_dev = NULL;

drivers/iommu/omap-iommu.c

1599

struct omap_iommu_device *iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1600

struct omap_iommu *oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1695

return &oiommu->iommu;

drivers/iommu/omap-iommu.c

952

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

959

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

960

iommu += (omap_domain->num_iommus - 1);

drivers/iommu/omap-iommu.c

961

for (i = 0; i < omap_domain->num_iommus; i++, iommu--) {

drivers/iommu/omap-iommu.c

962

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

981

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

988

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

989

for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

990

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.h

82

struct iommu_device iommu;

drivers/iommu/riscv/iommu-pci.c

101

riscv_iommu_remove(iommu);

drivers/iommu/riscv/iommu-pci.c

106

struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);

drivers/iommu/riscv/iommu-pci.c

108

riscv_iommu_disable(iommu);

drivers/iommu/riscv/iommu-pci.c

36

struct riscv_iommu_device *iommu;

drivers/iommu/riscv/iommu-pci.c

53

iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/riscv/iommu-pci.c

54

if (!iommu)

drivers/iommu/riscv/iommu-pci.c

57

iommu->dev = dev;

drivers/iommu/riscv/iommu-pci.c

58

iommu->reg = pcim_iomap_table(pdev)[0];

drivers/iommu/riscv/iommu-pci.c

61

dev_set_drvdata(dev, iommu);

drivers/iommu/riscv/iommu-pci.c

64

iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);

drivers/iommu/riscv/iommu-pci.c

65

iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);

drivers/iommu/riscv/iommu-pci.c

68

switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {

drivers/iommu/riscv/iommu-pci.c

84

iommu->irqs_count = rc;

drivers/iommu/riscv/iommu-pci.c

85

for (vec = 0; vec < iommu->irqs_count; vec++)

drivers/iommu/riscv/iommu-pci.c

86

iommu->irqs[vec] = msi_get_virq(dev, vec);

drivers/iommu/riscv/iommu-pci.c

89

if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {

drivers/iommu/riscv/iommu-pci.c

90

iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;

drivers/iommu/riscv/iommu-pci.c

91

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);

drivers/iommu/riscv/iommu-pci.c

94

return riscv_iommu_init(iommu);

drivers/iommu/riscv/iommu-pci.c

99

struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);

drivers/iommu/riscv/iommu-platform.c

102

for (vec = 0; vec < iommu->irqs_count; vec++)

drivers/iommu/riscv/iommu-platform.c

103

iommu->irqs[vec] = msi_get_virq(dev, vec);

drivers/iommu/riscv/iommu-platform.c

106

if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {

drivers/iommu/riscv/iommu-platform.c

107

iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;

drivers/iommu/riscv/iommu-platform.c

108

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);

drivers/iommu/riscv/iommu-platform.c

123

for (vec = 0; vec < iommu->irqs_count; vec++)

drivers/iommu/riscv/iommu-platform.c

124

iommu->irqs[vec] = platform_get_irq(pdev, vec);

drivers/iommu/riscv/iommu-platform.c

127

if (!(iommu->fctl & RISCV_IOMMU_FCTL_WSI)) {

drivers/iommu/riscv/iommu-platform.c

128

iommu->fctl |= RISCV_IOMMU_FCTL_WSI;

drivers/iommu/riscv/iommu-platform.c

129

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);

drivers/iommu/riscv/iommu-platform.c

137

return riscv_iommu_init(iommu);

drivers/iommu/riscv/iommu-platform.c

142

struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);

drivers/iommu/riscv/iommu-platform.c

143

bool msi = !(iommu->fctl & RISCV_IOMMU_FCTL_WSI);

drivers/iommu/riscv/iommu-platform.c

145

riscv_iommu_remove(iommu);

drivers/iommu/riscv/iommu-platform.c

27

struct riscv_iommu_device *iommu = dev_get_drvdata(dev);

drivers/iommu/riscv/iommu-platform.c

41

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_ADDR(idx), addr);

drivers/iommu/riscv/iommu-platform.c

42

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_DATA(idx), msg->data);

drivers/iommu/riscv/iommu-platform.c

43

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_CTRL(idx), 0);

drivers/iommu/riscv/iommu-platform.c

50

struct riscv_iommu_device *iommu = NULL;

drivers/iommu/riscv/iommu-platform.c

55

iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/riscv/iommu-platform.c

56

if (!iommu)

drivers/iommu/riscv/iommu-platform.c

59

iommu->dev = dev;

drivers/iommu/riscv/iommu-platform.c

60

iommu->reg = devm_platform_get_and_ioremap_resource(pdev, 0, &res);

drivers/iommu/riscv/iommu-platform.c

61

if (IS_ERR(iommu->reg))

drivers/iommu/riscv/iommu-platform.c

62

return dev_err_probe(dev, PTR_ERR(iommu->reg),

drivers/iommu/riscv/iommu-platform.c

65

dev_set_drvdata(dev, iommu);

drivers/iommu/riscv/iommu-platform.c

68

iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);

drivers/iommu/riscv/iommu-platform.c

69

iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);

drivers/iommu/riscv/iommu-platform.c

71

iommu->irqs_count = platform_irq_count(pdev);

drivers/iommu/riscv/iommu-platform.c

72

if (iommu->irqs_count <= 0)

drivers/iommu/riscv/iommu-platform.c

75

if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT)

drivers/iommu/riscv/iommu-platform.c

76

iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;

drivers/iommu/riscv/iommu-platform.c

78

igs = FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps);

drivers/iommu/riscv/iommu-platform.c

95

ret = platform_device_msi_init_and_alloc_irqs(dev, iommu->irqs_count,

drivers/iommu/riscv/iommu.c

100

devres_release(iommu->dev, riscv_iommu_devres_pages_release,

drivers/iommu/riscv/iommu.c

1012

static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

1023

dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);

drivers/iommu/riscv/iommu.c

1033

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

1038

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

1045

dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);

drivers/iommu/riscv/iommu.c

1058

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

1061

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

1308

static bool riscv_iommu_pt_supported(struct riscv_iommu_device *iommu, int pgd_mode)

drivers/iommu/riscv/iommu.c

131

static int riscv_iommu_queue_alloc(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

1312

return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39;

drivers/iommu/riscv/iommu.c

1315

return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48;

drivers/iommu/riscv/iommu.c

1318

return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57;

drivers/iommu/riscv/iommu.c

1328

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1332

if (!riscv_iommu_pt_supported(iommu, domain->pgd_mode))

drivers/iommu/riscv/iommu.c

1343

riscv_iommu_iodir_update(iommu, dev, fsc, ta);

drivers/iommu/riscv/iommu.c

1363

struct riscv_iommu_device *iommu;

drivers/iommu/riscv/iommu.c

1368

iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1369

if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57) {

drivers/iommu/riscv/iommu.c

1372

} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48) {

drivers/iommu/riscv/iommu.c

1375

} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39) {

drivers/iommu/riscv/iommu.c

1389

domain->numa_node = dev_to_node(iommu->dev);

drivers/iommu/riscv/iommu.c

1390

domain->amo_enabled = !!(iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD);

drivers/iommu/riscv/iommu.c

142

riscv_iommu_writeq(iommu, queue->qbr, RISCV_IOMMU_QUEUE_LOG2SZ_FIELD);

drivers/iommu/riscv/iommu.c

143

qb = riscv_iommu_readq(iommu, queue->qbr);

drivers/iommu/riscv/iommu.c

1433

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1437

riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, 0);

drivers/iommu/riscv/iommu.c

1455

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1458

riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, RISCV_IOMMU_PC_TA_V);

drivers/iommu/riscv/iommu.c

1487

struct riscv_iommu_device *iommu;

drivers/iommu/riscv/iommu.c

1496

iommu = dev_get_drvdata(fwspec->iommu_fwnode->dev);

drivers/iommu/riscv/iommu.c

1497

if (!iommu)

drivers/iommu/riscv/iommu.c

1504

if (iommu->ddt_mode <= RISCV_IOMMU_DDTP_IOMMU_MODE_BARE)

drivers/iommu/riscv/iommu.c

1515

if (iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD)

drivers/iommu/riscv/iommu.c

1518

dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);

drivers/iommu/riscv/iommu.c

1530

return &iommu->iommu;

drivers/iommu/riscv/iommu.c

1551

static int riscv_iommu_init_check(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

1560

ddtp = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_DDTP);

drivers/iommu/riscv/iommu.c

1568

riscv_iommu_disable(iommu);

drivers/iommu/riscv/iommu.c

1573

!!(iommu->fctl & RISCV_IOMMU_FCTL_BE)) {

drivers/iommu/riscv/iommu.c

1574

if (!(iommu->caps & RISCV_IOMMU_CAPABILITIES_END))

drivers/iommu/riscv/iommu.c

1576

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL,

drivers/iommu/riscv/iommu.c

1577

iommu->fctl ^ RISCV_IOMMU_FCTL_BE);

drivers/iommu/riscv/iommu.c

1578

iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);

drivers/iommu/riscv/iommu.c

1580

!!(iommu->fctl & RISCV_IOMMU_FCTL_BE))

drivers/iommu/riscv/iommu.c

1588

if (!iommu->irqs_count)

drivers/iommu/riscv/iommu.c

1591

iommu->icvec = FIELD_PREP(RISCV_IOMMU_ICVEC_FIV, 1 % iommu->irqs_count) |

drivers/iommu/riscv/iommu.c

1592

FIELD_PREP(RISCV_IOMMU_ICVEC_PIV, 2 % iommu->irqs_count) |

drivers/iommu/riscv/iommu.c

1593

FIELD_PREP(RISCV_IOMMU_ICVEC_PMIV, 3 % iommu->irqs_count);

drivers/iommu/riscv/iommu.c

1594

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_ICVEC, iommu->icvec);

drivers/iommu/riscv/iommu.c

1595

iommu->icvec = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_ICVEC);

drivers/iommu/riscv/iommu.c

1596

if (max3(FIELD_GET(RISCV_IOMMU_ICVEC_CIV, iommu->icvec),

drivers/iommu/riscv/iommu.c

1597

FIELD_GET(RISCV_IOMMU_ICVEC_FIV, iommu->icvec),

drivers/iommu/riscv/iommu.c

1598

max(FIELD_GET(RISCV_IOMMU_ICVEC_PIV, iommu->icvec),

drivers/iommu/riscv/iommu.c

1599

FIELD_GET(RISCV_IOMMU_ICVEC_PMIV, iommu->icvec))) >= iommu->irqs_count)

drivers/iommu/riscv/iommu.c

1605

void riscv_iommu_remove(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

1607

iommu_device_unregister(&iommu->iommu);

drivers/iommu/riscv/iommu.c

1608

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/riscv/iommu.c

1609

riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);

drivers/iommu/riscv/iommu.c

1610

riscv_iommu_queue_disable(&iommu->cmdq);

drivers/iommu/riscv/iommu.c

1611

riscv_iommu_queue_disable(&iommu->fltq);

drivers/iommu/riscv/iommu.c

1614

int riscv_iommu_init(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

1618

RISCV_IOMMU_QUEUE_INIT(&iommu->cmdq, CQ);

drivers/iommu/riscv/iommu.c

1619

RISCV_IOMMU_QUEUE_INIT(&iommu->fltq, FQ);

drivers/iommu/riscv/iommu.c

1621

rc = riscv_iommu_init_check(iommu);

drivers/iommu/riscv/iommu.c

1623

return dev_err_probe(iommu->dev, rc, "unexpected device state\n");

drivers/iommu/riscv/iommu.c

1625

rc = riscv_iommu_iodir_alloc(iommu);

drivers/iommu/riscv/iommu.c

1629

rc = riscv_iommu_queue_alloc(iommu, &iommu->cmdq,

drivers/iommu/riscv/iommu.c

163

queue->base = devm_ioremap(iommu->dev, queue->phys, queue_size);

drivers/iommu/riscv/iommu.c

1634

rc = riscv_iommu_queue_alloc(iommu, &iommu->fltq,

drivers/iommu/riscv/iommu.c

1639

rc = riscv_iommu_queue_enable(iommu, &iommu->cmdq, riscv_iommu_cmdq_process);

drivers/iommu/riscv/iommu.c

1643

rc = riscv_iommu_queue_enable(iommu, &iommu->fltq, riscv_iommu_fltq_process);

drivers/iommu/riscv/iommu.c

1647

rc = riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_MAX);

drivers/iommu/riscv/iommu.c

1651

rc = iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "riscv-iommu@%s",

drivers/iommu/riscv/iommu.c

1652

dev_name(iommu->dev));

drivers/iommu/riscv/iommu.c

1654

dev_err_probe(iommu->dev, rc, "cannot register sysfs interface\n");

drivers/iommu/riscv/iommu.c

1659

rc = rimt_iommu_register(iommu->dev);

drivers/iommu/riscv/iommu.c

1661

dev_err_probe(iommu->dev, rc, "cannot register iommu with RIMT\n");

drivers/iommu/riscv/iommu.c

1666

rc = iommu_device_register(&iommu->iommu, &riscv_iommu_ops, iommu->dev);

drivers/iommu/riscv/iommu.c

1668

dev_err_probe(iommu->dev, rc, "cannot register iommu interface\n");

drivers/iommu/riscv/iommu.c

1675

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/riscv/iommu.c

1677

riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);

drivers/iommu/riscv/iommu.c

1679

riscv_iommu_queue_disable(&iommu->fltq);

drivers/iommu/riscv/iommu.c

1680

riscv_iommu_queue_disable(&iommu->cmdq);

drivers/iommu/riscv/iommu.c

169

iommu, max(queue_size, SZ_4K));

drivers/iommu/riscv/iommu.c

181

riscv_iommu_writeq(iommu, queue->qbr, qb);

drivers/iommu/riscv/iommu.c

182

rb = riscv_iommu_readq(iommu, queue->qbr);

drivers/iommu/riscv/iommu.c

184

dev_err(iommu->dev, "queue #%u allocation failed\n", queue->qid);

drivers/iommu/riscv/iommu.c

191

dev_dbg(iommu->dev, "queue #%u allocated 2^%u entries",

drivers/iommu/riscv/iommu.c

202

if (riscv_iommu_readl(queue->iommu, RISCV_IOMMU_REG_IPSR) & Q_IPSR(queue))

drivers/iommu/riscv/iommu.c

208

static int riscv_iommu_queue_vec(struct riscv_iommu_device *iommu, int n)

drivers/iommu/riscv/iommu.c

211

return (iommu->icvec >> (n * 4)) & RISCV_IOMMU_ICVEC_CIV;

drivers/iommu/riscv/iommu.c

220

static int riscv_iommu_queue_enable(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

224

const unsigned int irq = iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)];

drivers/iommu/riscv/iommu.c

228

if (queue->iommu)

drivers/iommu/riscv/iommu.c

235

queue->iommu = iommu;

drivers/iommu/riscv/iommu.c

238

dev_name(iommu->dev), queue);

drivers/iommu/riscv/iommu.c

240

queue->iommu = NULL;

drivers/iommu/riscv/iommu.c

246

riscv_iommu_writel(queue->iommu, Q_TAIL(queue), 0);

drivers/iommu/riscv/iommu.c

248

riscv_iommu_writel(queue->iommu, Q_HEAD(queue), 0);

drivers/iommu/riscv/iommu.c

256

riscv_iommu_writel(iommu, queue->qcr,

drivers/iommu/riscv/iommu.c

261

riscv_iommu_readl_timeout(iommu, queue->qcr,

drivers/iommu/riscv/iommu.c

269

riscv_iommu_writel(iommu, queue->qcr, 0);

drivers/iommu/riscv/iommu.c

271

queue->iommu = NULL;

drivers/iommu/riscv/iommu.c

272

dev_err(iommu->dev, "queue #%u failed to start\n", queue->qid);

drivers/iommu/riscv/iommu.c

277

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));

drivers/iommu/riscv/iommu.c

288

struct riscv_iommu_device *iommu = queue->iommu;

drivers/iommu/riscv/iommu.c

291

if (!iommu)

drivers/iommu/riscv/iommu.c

294

free_irq(iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)], queue);

drivers/iommu/riscv/iommu.c

295

riscv_iommu_writel(iommu, queue->qcr, 0);

drivers/iommu/riscv/iommu.c

296

riscv_iommu_readl_timeout(iommu, queue->qcr,

drivers/iommu/riscv/iommu.c

301

dev_err(iommu->dev, "fail to disable hardware queue #%u, csr 0x%x\n",

drivers/iommu/riscv/iommu.c

304

queue->iommu = NULL;

drivers/iommu/riscv/iommu.c

325

if (riscv_iommu_readl_timeout(queue->iommu, Q_TAIL(queue),

drivers/iommu/riscv/iommu.c

328

dev_err_once(queue->iommu->dev,

drivers/iommu/riscv/iommu.c

347

riscv_iommu_writel(queue->iommu, Q_HEAD(queue), Q_ITEM(queue, head));

drivers/iommu/riscv/iommu.c

357

if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,

drivers/iommu/riscv/iommu.c

411

if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,

drivers/iommu/riscv/iommu.c

432

riscv_iommu_writel(queue->iommu, Q_TAIL(queue), Q_ITEM(queue, prod + 1));

drivers/iommu/riscv/iommu.c

44

iommu_get_iommu_dev(dev, struct riscv_iommu_device, iommu)

drivers/iommu/riscv/iommu.c

448

dev_err_once(queue->iommu->dev, "Hardware error: command enqueue failed\n");

drivers/iommu/riscv/iommu.c

464

ctrl = riscv_iommu_readl(queue->iommu, queue->qcr);

drivers/iommu/riscv/iommu.c

467

riscv_iommu_writel(queue->iommu, queue->qcr, ctrl);

drivers/iommu/riscv/iommu.c

468

dev_warn(queue->iommu->dev,

drivers/iommu/riscv/iommu.c

480

riscv_iommu_writel(queue->iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));

drivers/iommu/riscv/iommu.c

486

static void riscv_iommu_cmd_send(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

489

riscv_iommu_queue_send(&iommu->cmdq, cmd, sizeof(*cmd));

drivers/iommu/riscv/iommu.c

493

static void riscv_iommu_cmd_sync(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

500

prod = riscv_iommu_queue_send(&iommu->cmdq, &cmd, sizeof(cmd));

drivers/iommu/riscv/iommu.c

505

if (riscv_iommu_queue_wait(&iommu->cmdq, prod, timeout_us))

drivers/iommu/riscv/iommu.c

506

dev_err_once(iommu->dev,

drivers/iommu/riscv/iommu.c

514

static void riscv_iommu_fault(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

522

dev_warn_ratelimited(iommu->dev,

drivers/iommu/riscv/iommu.c

531

struct riscv_iommu_device *iommu = queue->iommu;

drivers/iommu/riscv/iommu.c

539

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));

drivers/iommu/riscv/iommu.c

544

riscv_iommu_fault(iommu, &events[Q_ITEM(queue, idx)]);

drivers/iommu/riscv/iommu.c

549

ctrl = riscv_iommu_readl(iommu, queue->qcr);

drivers/iommu/riscv/iommu.c

551

riscv_iommu_writel(iommu, queue->qcr, ctrl);

drivers/iommu/riscv/iommu.c

552

dev_warn(iommu->dev,

drivers/iommu/riscv/iommu.c

563

static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

566

const bool base_format = !(iommu->caps & RISCV_IOMMU_CAPABILITIES_MSI_FLAT);

drivers/iommu/riscv/iommu.c

574

if (iommu->ddt_mode < RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL ||

drivers/iommu/riscv/iommu.c

575

iommu->ddt_mode > RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL)

drivers/iommu/riscv/iommu.c

600

depth = iommu->ddt_mode - RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL;

drivers/iommu/riscv/iommu.c

605

for (ddtp = iommu->ddt_root; depth-- > 0;) {

drivers/iommu/riscv/iommu.c

624

ptr = riscv_iommu_get_pages(iommu, SZ_4K);

drivers/iommu/riscv/iommu.c

637

riscv_iommu_free_pages(iommu, ptr);

drivers/iommu/riscv/iommu.c

655

void riscv_iommu_disable(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

657

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,

drivers/iommu/riscv/iommu.c

660

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_CQCSR, 0);

drivers/iommu/riscv/iommu.c

661

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FQCSR, 0);

drivers/iommu/riscv/iommu.c

662

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_PQCSR, 0);

drivers/iommu/riscv/iommu.c

665

#define riscv_iommu_read_ddtp(iommu) ({ \

drivers/iommu/riscv/iommu.c

667

riscv_iommu_readq_timeout((iommu), RISCV_IOMMU_REG_DDTP, ddtp, \

drivers/iommu/riscv/iommu.c

672

static int riscv_iommu_iodir_alloc(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

677

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

689

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,

drivers/iommu/riscv/iommu.c

691

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

695

iommu->ddt_phys = ppn_to_phys(ddtp);

drivers/iommu/riscv/iommu.c

696

if (iommu->ddt_phys)

drivers/iommu/riscv/iommu.c

697

iommu->ddt_root = devm_ioremap(iommu->dev,

drivers/iommu/riscv/iommu.c

698

iommu->ddt_phys, PAGE_SIZE);

drivers/iommu/riscv/iommu.c

699

if (iommu->ddt_root)

drivers/iommu/riscv/iommu.c

70

static void *riscv_iommu_get_pages(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

700

memset(iommu->ddt_root, 0, PAGE_SIZE);

drivers/iommu/riscv/iommu.c

703

if (!iommu->ddt_root) {

drivers/iommu/riscv/iommu.c

704

iommu->ddt_root = riscv_iommu_get_pages(iommu, SZ_4K);

drivers/iommu/riscv/iommu.c

705

iommu->ddt_phys = __pa(iommu->ddt_root);

drivers/iommu/riscv/iommu.c

708

if (!iommu->ddt_root)

drivers/iommu/riscv/iommu.c

719

static int riscv_iommu_iodir_set_mode(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

722

struct device *dev = iommu->dev;

drivers/iommu/riscv/iommu.c

727

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

742

rq_ddtp |= phys_to_ppn(iommu->ddt_phys);

drivers/iommu/riscv/iommu.c

744

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP, rq_ddtp);

drivers/iommu/riscv/iommu.c

745

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

76

addr = iommu_alloc_pages_node_sz(dev_to_node(iommu->dev),

drivers/iommu/riscv/iommu.c

789

iommu->ddt_mode = mode;

drivers/iommu/riscv/iommu.c

795

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

799

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

802

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

852

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

868

if (dev_to_iommu(list_entry(bonds, struct riscv_iommu_bond, list)->dev) == iommu)

drivers/iommu/riscv/iommu.c

882

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

896

else if (dev_to_iommu(bond->dev) == iommu)

drivers/iommu/riscv/iommu.c

91

devres_add(iommu->dev, devres);

drivers/iommu/riscv/iommu.c

911

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

913

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

929

struct riscv_iommu_device *iommu, *prev;

drivers/iommu/riscv/iommu.c

96

static void riscv_iommu_free_pages(struct riscv_iommu_device *iommu, void *addr)

drivers/iommu/riscv/iommu.c

962

iommu = dev_to_iommu(bond->dev);

drivers/iommu/riscv/iommu.c

970

if (iommu == prev)

drivers/iommu/riscv/iommu.c

978

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

981

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

983

prev = iommu;

drivers/iommu/riscv/iommu.c

988

iommu = dev_to_iommu(bond->dev);

drivers/iommu/riscv/iommu.c

989

if (iommu == prev)

drivers/iommu/riscv/iommu.c

992

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

993

prev = iommu;

drivers/iommu/riscv/iommu.h

28

struct riscv_iommu_device *iommu; /* iommu device handling the queue when active */

drivers/iommu/riscv/iommu.h

38

struct iommu_device iommu;

drivers/iommu/riscv/iommu.h

65

int riscv_iommu_init(struct riscv_iommu_device *iommu);

drivers/iommu/riscv/iommu.h

66

void riscv_iommu_remove(struct riscv_iommu_device *iommu);

drivers/iommu/riscv/iommu.h

67

void riscv_iommu_disable(struct riscv_iommu_device *iommu);

drivers/iommu/riscv/iommu.h

69

#define riscv_iommu_readl(iommu, addr) \

drivers/iommu/riscv/iommu.h

70

readl_relaxed((iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

72

#define riscv_iommu_readq(iommu, addr) \

drivers/iommu/riscv/iommu.h

73

readq_relaxed((iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

75

#define riscv_iommu_writel(iommu, addr, val) \

drivers/iommu/riscv/iommu.h

76

writel_relaxed((val), (iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

78

#define riscv_iommu_writeq(iommu, addr, val) \

drivers/iommu/riscv/iommu.h

79

writeq_relaxed((val), (iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

81

#define riscv_iommu_readq_timeout(iommu, addr, val, cond, delay_us, timeout_us) \

drivers/iommu/riscv/iommu.h

82

readx_poll_timeout(readq_relaxed, (iommu)->reg + (addr), val, cond, \

drivers/iommu/riscv/iommu.h

85

#define riscv_iommu_readl_timeout(iommu, addr, val, cond, delay_us, timeout_us) \

drivers/iommu/riscv/iommu.h

86

readx_poll_timeout(readl_relaxed, (iommu)->reg + (addr), val, cond, \

drivers/iommu/rockchip-iommu.c

1011

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1020

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

1021

if (!iommu)

drivers/iommu/rockchip-iommu.c

1027

if (iommu->domain == domain)

drivers/iommu/rockchip-iommu.c

1034

iommu->domain = domain;

drivers/iommu/rockchip-iommu.c

1037

list_add_tail(&iommu->node, &rk_domain->iommus);

drivers/iommu/rockchip-iommu.c

1040

ret = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/rockchip-iommu.c

1044

ret = rk_iommu_enable(iommu);

drivers/iommu/rockchip-iommu.c

1054

iommu->domain));

drivers/iommu/rockchip-iommu.c

1057

pm_runtime_put(iommu->dev);

drivers/iommu/rockchip-iommu.c

1065

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1081

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

1082

rk_domain->dma_dev = iommu->dev;

drivers/iommu/rockchip-iommu.c

1138

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1144

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

1146

data->link = device_link_add(dev, iommu->dev,

drivers/iommu/rockchip-iommu.c

1149

return &iommu->iommu;

drivers/iommu/rockchip-iommu.c

117

struct iommu_device iommu;

drivers/iommu/rockchip-iommu.c

1171

data->iommu = platform_get_drvdata(iommu_dev);

drivers/iommu/rockchip-iommu.c

1198

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1204

iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/rockchip-iommu.c

1205

if (!iommu)

drivers/iommu/rockchip-iommu.c

1208

iommu->domain = &rk_identity_domain;

drivers/iommu/rockchip-iommu.c

1210

platform_set_drvdata(pdev, iommu);

drivers/iommu/rockchip-iommu.c

1211

iommu->dev = dev;

drivers/iommu/rockchip-iommu.c

1212

iommu->num_mmu = 0;

drivers/iommu/rockchip-iommu.c

1225

iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases),

drivers/iommu/rockchip-iommu.c

1227

if (!iommu->bases)

drivers/iommu/rockchip-iommu.c

1234

iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res);

drivers/iommu/rockchip-iommu.c

1235

if (IS_ERR(iommu->bases[i]))

drivers/iommu/rockchip-iommu.c

1237

iommu->num_mmu++;

drivers/iommu/rockchip-iommu.c

1239

if (iommu->num_mmu == 0)

drivers/iommu/rockchip-iommu.c

124

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1240

return PTR_ERR(iommu->bases[0]);

drivers/iommu/rockchip-iommu.c

1242

iommu->num_irq = platform_irq_count(pdev);

drivers/iommu/rockchip-iommu.c

1243

if (iommu->num_irq < 0)

drivers/iommu/rockchip-iommu.c

1244

return iommu->num_irq;

drivers/iommu/rockchip-iommu.c

1246

iommu->reset_disabled = device_property_read_bool(dev,

drivers/iommu/rockchip-iommu.c

1249

iommu->num_clocks = ARRAY_SIZE(rk_iommu_clocks);

drivers/iommu/rockchip-iommu.c

1250

iommu->clocks = devm_kcalloc(iommu->dev, iommu->num_clocks,

drivers/iommu/rockchip-iommu.c

1251

sizeof(*iommu->clocks), GFP_KERNEL);

drivers/iommu/rockchip-iommu.c

1252

if (!iommu->clocks)

drivers/iommu/rockchip-iommu.c

1255

for (i = 0; i < iommu->num_clocks; ++i)

drivers/iommu/rockchip-iommu.c

1256

iommu->clocks[i].id = rk_iommu_clocks[i];

drivers/iommu/rockchip-iommu.c

1263

err = devm_clk_bulk_get(iommu->dev, iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

1265

iommu->num_clocks = 0;

drivers/iommu/rockchip-iommu.c

1269

err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

1275

for (i = 0; i < iommu->num_irq; i++) {

drivers/iommu/rockchip-iommu.c

1283

err = devm_request_irq(iommu->dev, irq, rk_iommu_irq,

drivers/iommu/rockchip-iommu.c

1284

IRQF_SHARED, dev_name(dev), iommu);

drivers/iommu/rockchip-iommu.c

1291

err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));

drivers/iommu/rockchip-iommu.c

1295

err = iommu_device_register(&iommu->iommu, &rk_iommu_ops, dev);

drivers/iommu/rockchip-iommu.c

1301

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/rockchip-iommu.c

1304

clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

1310

struct rk_iommu *iommu = platform_get_drvdata(pdev);

drivers/iommu/rockchip-iommu.c

1313

for (i = 0; i < iommu->num_irq; i++) {

drivers/iommu/rockchip-iommu.c

1316

devm_free_irq(iommu->dev, irq, iommu);

drivers/iommu/rockchip-iommu.c

1324

struct rk_iommu *iommu = dev_get_drvdata(dev);

drivers/iommu/rockchip-iommu.c

1326

if (iommu->domain == &rk_identity_domain)

drivers/iommu/rockchip-iommu.c

1329

rk_iommu_disable(iommu);

drivers/iommu/rockchip-iommu.c

1335

struct rk_iommu *iommu = dev_get_drvdata(dev);

drivers/iommu/rockchip-iommu.c

1337

if (iommu->domain == &rk_identity_domain)

drivers/iommu/rockchip-iommu.c

1340

return rk_iommu_enable(iommu);

drivers/iommu/rockchip-iommu.c

347

static void rk_iommu_command(struct rk_iommu *iommu, u32 command)

drivers/iommu/rockchip-iommu.c

351

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

352

writel(command, iommu->bases[i] + RK_MMU_COMMAND);

drivers/iommu/rockchip-iommu.c

359

static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova_start,

drivers/iommu/rockchip-iommu.c

368

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

372

rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova);

drivers/iommu/rockchip-iommu.c

376

static bool rk_iommu_is_stall_active(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

381

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

382

active &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &

drivers/iommu/rockchip-iommu.c

388

static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

393

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

394

enable &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &

drivers/iommu/rockchip-iommu.c

400

static bool rk_iommu_is_reset_done(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

405

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

406

done &= rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0;

drivers/iommu/rockchip-iommu.c

411

static int rk_iommu_enable_stall(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

416

if (rk_iommu_is_stall_active(iommu))

drivers/iommu/rockchip-iommu.c

420

if (!rk_iommu_is_paging_enabled(iommu))

drivers/iommu/rockchip-iommu.c

423

rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL);

drivers/iommu/rockchip-iommu.c

425

ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val,

drivers/iommu/rockchip-iommu.c

429

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

430

dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

431

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

436

static int rk_iommu_disable_stall(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

441

if (!rk_iommu_is_stall_active(iommu))

drivers/iommu/rockchip-iommu.c

444

rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL);

drivers/iommu/rockchip-iommu.c

446

ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val,

drivers/iommu/rockchip-iommu.c

450

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

451

dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

452

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

457

static int rk_iommu_enable_paging(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

462

if (rk_iommu_is_paging_enabled(iommu))

drivers/iommu/rockchip-iommu.c

465

rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING);

drivers/iommu/rockchip-iommu.c

467

ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val,

drivers/iommu/rockchip-iommu.c

471

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

472

dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

473

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

478

static int rk_iommu_disable_paging(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

483

if (!rk_iommu_is_paging_enabled(iommu))

drivers/iommu/rockchip-iommu.c

486

rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING);

drivers/iommu/rockchip-iommu.c

488

ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val,

drivers/iommu/rockchip-iommu.c

492

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

493

dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

494

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

499

static int rk_iommu_force_reset(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

505

if (iommu->reset_disabled)

drivers/iommu/rockchip-iommu.c

512

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

514

rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr);

drivers/iommu/rockchip-iommu.c

516

if (dte_addr != rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR)) {

drivers/iommu/rockchip-iommu.c

517

dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n");

drivers/iommu/rockchip-iommu.c

522

rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET);

drivers/iommu/rockchip-iommu.c

524

ret = readx_poll_timeout(rk_iommu_is_reset_done, iommu, val,

drivers/iommu/rockchip-iommu.c

528

dev_err(iommu->dev, "FORCE_RESET command timed out\n");

drivers/iommu/rockchip-iommu.c

535

static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)

drivers/iommu/rockchip-iommu.c

537

void __iomem *base = iommu->bases[index];

drivers/iommu/rockchip-iommu.c

574

dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n",

drivers/iommu/rockchip-iommu.c

576

dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n",

drivers/iommu/rockchip-iommu.c

584

struct rk_iommu *iommu = dev_id;

drivers/iommu/rockchip-iommu.c

591

err = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/rockchip-iommu.c

595

if (WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)))

drivers/iommu/rockchip-iommu.c

598

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

599

int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS);

drivers/iommu/rockchip-iommu.c

604

iova = rk_iommu_read(iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR);

drivers/iommu/rockchip-iommu.c

609

status = rk_iommu_read(iommu->bases[i], RK_MMU_STATUS);

drivers/iommu/rockchip-iommu.c

613

dev_err(iommu->dev, "Page fault at %pad of type %s\n",

drivers/iommu/rockchip-iommu.c

617

log_iova(iommu, i, iova);

drivers/iommu/rockchip-iommu.c

624

if (iommu->domain != &rk_identity_domain)

drivers/iommu/rockchip-iommu.c

625

report_iommu_fault(iommu->domain, iommu->dev, iova,

drivers/iommu/rockchip-iommu.c

628

dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n");

drivers/iommu/rockchip-iommu.c

630

rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);

drivers/iommu/rockchip-iommu.c

631

rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE);

drivers/iommu/rockchip-iommu.c

635

dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova);

drivers/iommu/rockchip-iommu.c

638

dev_err(iommu->dev, "unexpected int_status: %#08x\n",

drivers/iommu/rockchip-iommu.c

641

rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status);

drivers/iommu/rockchip-iommu.c

644

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

647

pm_runtime_put(iommu->dev);

drivers/iommu/rockchip-iommu.c

688

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

691

iommu = list_entry(pos, struct rk_iommu, node);

drivers/iommu/rockchip-iommu.c

694

ret = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/rockchip-iommu.c

698

WARN_ON(clk_bulk_enable(iommu->num_clocks,

drivers/iommu/rockchip-iommu.c

699

iommu->clocks));

drivers/iommu/rockchip-iommu.c

700

rk_iommu_zap_lines(iommu, iova, size);

drivers/iommu/rockchip-iommu.c

701

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

702

pm_runtime_put(iommu->dev);

drivers/iommu/rockchip-iommu.c

907

return data ? data->iommu : NULL;

drivers/iommu/rockchip-iommu.c

911

static void rk_iommu_disable(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

916

WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks));

drivers/iommu/rockchip-iommu.c

917

rk_iommu_enable_stall(iommu);

drivers/iommu/rockchip-iommu.c

918

rk_iommu_disable_paging(iommu);

drivers/iommu/rockchip-iommu.c

919

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

920

rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0);

drivers/iommu/rockchip-iommu.c

921

rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0);

drivers/iommu/rockchip-iommu.c

923

rk_iommu_disable_stall(iommu);

drivers/iommu/rockchip-iommu.c

924

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

928

static int rk_iommu_enable(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

930

struct iommu_domain *domain = iommu->domain;

drivers/iommu/rockchip-iommu.c

934

ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

938

ret = rk_iommu_enable_stall(iommu);

drivers/iommu/rockchip-iommu.c

942

ret = rk_iommu_force_reset(iommu);

drivers/iommu/rockchip-iommu.c

946

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

947

rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,

drivers/iommu/rockchip-iommu.c

949

rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);

drivers/iommu/rockchip-iommu.c

950

rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);

drivers/iommu/rockchip-iommu.c

953

ret = rk_iommu_enable_paging(iommu);

drivers/iommu/rockchip-iommu.c

956

rk_iommu_disable_stall(iommu);

drivers/iommu/rockchip-iommu.c

958

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

966

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

972

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

973

if (!iommu)

drivers/iommu/rockchip-iommu.c

976

rk_domain = to_rk_domain(iommu->domain);

drivers/iommu/rockchip-iommu.c

980

if (iommu->domain == identity_domain)

drivers/iommu/rockchip-iommu.c

983

iommu->domain = identity_domain;

drivers/iommu/rockchip-iommu.c

986

list_del_init(&iommu->node);

drivers/iommu/rockchip-iommu.c

989

ret = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/rockchip-iommu.c

992

rk_iommu_disable(iommu);

drivers/iommu/rockchip-iommu.c

993

pm_runtime_put(iommu->dev);

drivers/iommu/s390-iommu.c

780

dev->iommu->shadow_on_flush = 1;

drivers/iommu/sprd-iommu.c

393

return &sdev->iommu;

drivers/iommu/sprd-iommu.c

486

ret = iommu_device_sysfs_add(&sdev->iommu, dev, NULL, dev_name(dev));

drivers/iommu/sprd-iommu.c

490

ret = iommu_device_register(&sdev->iommu, &sprd_iommu_ops, dev);

drivers/iommu/sprd-iommu.c

510

iommu_device_unregister(&sdev->iommu);

drivers/iommu/sprd-iommu.c

512

iommu_device_sysfs_remove(&sdev->iommu);

drivers/iommu/sprd-iommu.c

525

iommu_device_sysfs_remove(&sdev->iommu);

drivers/iommu/sprd-iommu.c

526

iommu_device_unregister(&sdev->iommu);

drivers/iommu/sprd-iommu.c

72

struct iommu_device iommu;

drivers/iommu/sun50i-iommu.c

1001

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

1004

iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/sun50i-iommu.c

1005

if (!iommu)

drivers/iommu/sun50i-iommu.c

1007

spin_lock_init(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

1008

iommu->domain = &sun50i_iommu_identity_domain;

drivers/iommu/sun50i-iommu.c

1009

platform_set_drvdata(pdev, iommu);

drivers/iommu/sun50i-iommu.c

101

struct iommu_device iommu;

drivers/iommu/sun50i-iommu.c

1010

iommu->dev = &pdev->dev;

drivers/iommu/sun50i-iommu.c

1012

iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),

drivers/iommu/sun50i-iommu.c

1016

if (!iommu->pt_pool)

drivers/iommu/sun50i-iommu.c

1019

iommu->base = devm_platform_ioremap_resource(pdev, 0);

drivers/iommu/sun50i-iommu.c

1020

if (IS_ERR(iommu->base)) {

drivers/iommu/sun50i-iommu.c

1021

ret = PTR_ERR(iommu->base);

drivers/iommu/sun50i-iommu.c

1031

iommu->clk = devm_clk_get(&pdev->dev, NULL);

drivers/iommu/sun50i-iommu.c

1032

if (IS_ERR(iommu->clk)) {

drivers/iommu/sun50i-iommu.c

1034

ret = PTR_ERR(iommu->clk);

drivers/iommu/sun50i-iommu.c

1038

iommu->reset = devm_reset_control_get(&pdev->dev, NULL);

drivers/iommu/sun50i-iommu.c

1039

if (IS_ERR(iommu->reset)) {

drivers/iommu/sun50i-iommu.c

1041

ret = PTR_ERR(iommu->reset);

drivers/iommu/sun50i-iommu.c

1045

ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,

drivers/iommu/sun50i-iommu.c

1050

ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);

drivers/iommu/sun50i-iommu.c

1055

dev_name(&pdev->dev), iommu);

drivers/iommu/sun50i-iommu.c

1062

iommu_device_unregister(&iommu->iommu);

drivers/iommu/sun50i-iommu.c

1065

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/sun50i-iommu.c

1068

kmem_cache_destroy(iommu->pt_pool);

drivers/iommu/sun50i-iommu.c

125

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

138

static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)

drivers/iommu/sun50i-iommu.c

140

return readl(iommu->base + offset);

drivers/iommu/sun50i-iommu.c

143

static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)

drivers/iommu/sun50i-iommu.c

145

writel(value, iommu->base + offset);

drivers/iommu/sun50i-iommu.c

294

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

298

dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);

drivers/iommu/sun50i-iommu.c

301

static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

307

iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);

drivers/iommu/sun50i-iommu.c

308

iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));

drivers/iommu/sun50i-iommu.c

309

iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

312

ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

315

dev_warn(iommu->dev, "TLB invalidation timed out!\n");

drivers/iommu/sun50i-iommu.c

318

static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

324

iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);

drivers/iommu/sun50i-iommu.c

325

iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

328

ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

331

dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");

drivers/iommu/sun50i-iommu.c

334

static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

337

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

339

iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);

drivers/iommu/sun50i-iommu.c

341

sun50i_iommu_zap_iova(iommu, iova);

drivers/iommu/sun50i-iommu.c

342

sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);

drivers/iommu/sun50i-iommu.c

344

sun50i_iommu_zap_iova(iommu, iova + size);

drivers/iommu/sun50i-iommu.c

345

sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);

drivers/iommu/sun50i-iommu.c

347

sun50i_iommu_zap_ptw_cache(iommu, iova);

drivers/iommu/sun50i-iommu.c

348

sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);

drivers/iommu/sun50i-iommu.c

350

sun50i_iommu_zap_ptw_cache(iommu, iova + size);

drivers/iommu/sun50i-iommu.c

351

sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);

drivers/iommu/sun50i-iommu.c

354

iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);

drivers/iommu/sun50i-iommu.c

357

static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

362

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

364

iommu_write(iommu,

drivers/iommu/sun50i-iommu.c

375

ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,

drivers/iommu/sun50i-iommu.c

379

dev_warn(iommu->dev, "TLB Flush timed out!\n");

drivers/iommu/sun50i-iommu.c

387

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

398

if (!iommu)

drivers/iommu/sun50i-iommu.c

401

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

402

sun50i_iommu_flush_all_tlb(iommu);

drivers/iommu/sun50i-iommu.c

403

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

410

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

413

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

414

sun50i_iommu_zap_range(iommu, iova, size);

drivers/iommu/sun50i-iommu.c

415

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

426

static int sun50i_iommu_enable(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

432

if (!iommu->domain)

drivers/iommu/sun50i-iommu.c

435

sun50i_domain = to_sun50i_domain(iommu->domain);

drivers/iommu/sun50i-iommu.c

437

ret = reset_control_deassert(iommu->reset);

drivers/iommu/sun50i-iommu.c

441

ret = clk_prepare_enable(iommu->clk);

drivers/iommu/sun50i-iommu.c

445

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

447

iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);

drivers/iommu/sun50i-iommu.c

448

iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,

drivers/iommu/sun50i-iommu.c

455

iommu_write(iommu, IOMMU_BYPASS_REG, 0);

drivers/iommu/sun50i-iommu.c

456

iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);

drivers/iommu/sun50i-iommu.c

457

iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),

drivers/iommu/sun50i-iommu.c

471

iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),

drivers/iommu/sun50i-iommu.c

479

iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),

drivers/iommu/sun50i-iommu.c

487

ret = sun50i_iommu_flush_all_tlb(iommu);

drivers/iommu/sun50i-iommu.c

489

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

493

iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);

drivers/iommu/sun50i-iommu.c

494

iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);

drivers/iommu/sun50i-iommu.c

496

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

501

clk_disable_unprepare(iommu->clk);

drivers/iommu/sun50i-iommu.c

504

reset_control_assert(iommu->reset);

drivers/iommu/sun50i-iommu.c

509

static void sun50i_iommu_disable(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

513

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

515

iommu_write(iommu, IOMMU_ENABLE_REG, 0);

drivers/iommu/sun50i-iommu.c

516

iommu_write(iommu, IOMMU_TTB_REG, 0);

drivers/iommu/sun50i-iommu.c

518

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

520

clk_disable_unprepare(iommu->clk);

drivers/iommu/sun50i-iommu.c

521

reset_control_assert(iommu->reset);

drivers/iommu/sun50i-iommu.c

524

static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

530

page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);

drivers/iommu/sun50i-iommu.c

534

pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);

drivers/iommu/sun50i-iommu.c

535

if (dma_mapping_error(iommu->dev, pt_dma)) {

drivers/iommu/sun50i-iommu.c

536

dev_err(iommu->dev, "Couldn't map L2 Page Table\n");

drivers/iommu/sun50i-iommu.c

537

kmem_cache_free(iommu->pt_pool, page_table);

drivers/iommu/sun50i-iommu.c

547

static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

552

dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);

drivers/iommu/sun50i-iommu.c

553

kmem_cache_free(iommu->pt_pool, page_table);

drivers/iommu/sun50i-iommu.c

559

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

572

page_table = sun50i_iommu_alloc_page_table(iommu, gfp);

drivers/iommu/sun50i-iommu.c

586

sun50i_iommu_free_page_table(iommu, drop_pt);

drivers/iommu/sun50i-iommu.c

600

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

608

dev_warn_once(iommu->dev,

drivers/iommu/sun50i-iommu.c

623

dev_err(iommu->dev,

drivers/iommu/sun50i-iommu.c

724

static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

727

iommu->domain = &sun50i_domain->domain;

drivers/iommu/sun50i-iommu.c

728

sun50i_domain->iommu = iommu;

drivers/iommu/sun50i-iommu.c

730

sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,

drivers/iommu/sun50i-iommu.c

732

if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {

drivers/iommu/sun50i-iommu.c

733

dev_err(iommu->dev, "Couldn't map L1 Page Table\n");

drivers/iommu/sun50i-iommu.c

737

return sun50i_iommu_enable(iommu);

drivers/iommu/sun50i-iommu.c

740

static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

761

sun50i_iommu_free_page_table(iommu, page_table);

drivers/iommu/sun50i-iommu.c

765

sun50i_iommu_disable(iommu);

drivers/iommu/sun50i-iommu.c

767

dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),

drivers/iommu/sun50i-iommu.c

770

iommu->domain = NULL;

drivers/iommu/sun50i-iommu.c

777

struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);

drivers/iommu/sun50i-iommu.c

782

if (iommu->domain == identity_domain)

drivers/iommu/sun50i-iommu.c

785

sun50i_domain = to_sun50i_domain(iommu->domain);

drivers/iommu/sun50i-iommu.c

787

sun50i_iommu_detach_domain(iommu, sun50i_domain);

drivers/iommu/sun50i-iommu.c

805

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

807

iommu = sun50i_iommu_from_dev(dev);

drivers/iommu/sun50i-iommu.c

808

if (!iommu)

drivers/iommu/sun50i-iommu.c

815

if (iommu->domain == domain)

drivers/iommu/sun50i-iommu.c

820

sun50i_iommu_attach_domain(iommu, sun50i_domain);

drivers/iommu/sun50i-iommu.c

827

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

829

iommu = sun50i_iommu_from_dev(dev);

drivers/iommu/sun50i-iommu.c

830

if (!iommu)

drivers/iommu/sun50i-iommu.c

833

return &iommu->iommu;

drivers/iommu/sun50i-iommu.c

867

static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

871

dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",

drivers/iommu/sun50i-iommu.c

874

if (iommu->domain)

drivers/iommu/sun50i-iommu.c

875

report_iommu_fault(iommu->domain, iommu->dev, iova, prot);

drivers/iommu/sun50i-iommu.c

877

dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");

drivers/iommu/sun50i-iommu.c

879

sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);

drivers/iommu/sun50i-iommu.c

882

static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

890

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

892

iova = iommu_read(iommu, addr_reg);

drivers/iommu/sun50i-iommu.c

893

blame = iommu_read(iommu, blame_reg);

drivers/iommu/sun50i-iommu.c

901

sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);

drivers/iommu/sun50i-iommu.c

906

static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

914

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

916

blame = iommu_read(iommu, IOMMU_INT_STA_REG);

drivers/iommu/sun50i-iommu.c

918

iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));

drivers/iommu/sun50i-iommu.c

919

aci = sun50i_get_pte_aci(iommu_read(iommu,

drivers/iommu/sun50i-iommu.c

956

sun50i_iommu_report_fault(iommu, master, iova, dir);

drivers/iommu/sun50i-iommu.c

964

struct sun50i_iommu *iommu = dev_id;

drivers/iommu/sun50i-iommu.c

966

spin_lock(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

968

status = iommu_read(iommu, IOMMU_INT_STA_REG);

drivers/iommu/sun50i-iommu.c

970

spin_unlock(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

974

l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);

drivers/iommu/sun50i-iommu.c

975

l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);

drivers/iommu/sun50i-iommu.c

978

sun50i_iommu_handle_pt_irq(iommu,

drivers/iommu/sun50i-iommu.c

982

sun50i_iommu_handle_pt_irq(iommu,

drivers/iommu/sun50i-iommu.c

986

sun50i_iommu_handle_perm_irq(iommu);

drivers/iommu/sun50i-iommu.c

988

iommu_write(iommu, IOMMU_INT_CLR_REG, status);

drivers/iommu/sun50i-iommu.c

991

iommu_write(iommu, IOMMU_RESET_REG, ~resets);

drivers/iommu/sun50i-iommu.c

992

iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);

drivers/iommu/sun50i-iommu.c

994

spin_unlock(&iommu->iommu_lock);

drivers/iommu/tegra-smmu.c

1169

err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));

drivers/iommu/tegra-smmu.c

1173

err = iommu_device_register(&smmu->iommu, &tegra_smmu_ops, dev);

drivers/iommu/tegra-smmu.c

1175

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/tegra-smmu.c

1187

iommu_device_unregister(&smmu->iommu);

drivers/iommu/tegra-smmu.c

1188

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/tegra-smmu.c

51

struct iommu_device iommu; /* IOMMU Core code handle */

drivers/iommu/tegra-smmu.c

843

const struct iommu_ops *ops = smmu->iommu.ops;

drivers/iommu/tegra-smmu.c

889

return &smmu->iommu;

drivers/iommu/virtio-iommu.c

1047

return &viommu->iommu;

drivers/iommu/virtio-iommu.c

1232

ret = iommu_device_sysfs_add(&viommu->iommu, dev, NULL, "%s",

drivers/iommu/virtio-iommu.c

1239

iommu_device_register(&viommu->iommu, &viommu_ops, parent_dev);

drivers/iommu/virtio-iommu.c

1257

iommu_device_sysfs_remove(&viommu->iommu);

drivers/iommu/virtio-iommu.c

1258

iommu_device_unregister(&viommu->iommu);

drivers/iommu/virtio-iommu.c

35

struct iommu_device iommu;

drivers/media/platform/qcom/venus/firmware.c

142

struct iommu_domain *iommu;

drivers/media/platform/qcom/venus/firmware.c

150

iommu = core->fw.iommu_domain;

drivers/media/platform/qcom/venus/firmware.c

153

ret = iommu_map(iommu, VENUS_FW_START_ADDR, mem_phys, mem_size,

drivers/media/platform/qcom/venus/firmware.c

168

struct iommu_domain *iommu;

drivers/media/platform/qcom/venus/firmware.c

187

iommu = core->fw.iommu_domain;

drivers/media/platform/qcom/venus/firmware.c

189

if (core->fw.mapped_mem_size && iommu) {

drivers/media/platform/qcom/venus/firmware.c

190

unmapped = iommu_unmap(iommu, VENUS_FW_START_ADDR, mapped);

drivers/media/platform/qcom/venus/firmware.c

369

struct iommu_domain *iommu;

drivers/media/platform/qcom/venus/firmware.c

374

iommu = core->fw.iommu_domain;

drivers/media/platform/qcom/venus/firmware.c

376

iommu_detach_device(iommu, core->fw.dev);

drivers/media/platform/qcom/venus/firmware.c

379

iommu_domain_free(iommu);

drivers/net/wireless/ath/ath10k/snoc.c

1694

struct iommu_domain *iommu;

drivers/net/wireless/ath/ath10k/snoc.c

1700

iommu = ar_snoc->fw.iommu_domain;

drivers/net/wireless/ath/ath10k/snoc.c

1702

unmapped_size = iommu_unmap(iommu, ar_snoc->fw.fw_start_addr,

drivers/net/wireless/ath/ath10k/snoc.c

1708

iommu_detach_device(iommu, ar_snoc->fw.dev);

drivers/net/wireless/ath/ath10k/snoc.c

1709

iommu_domain_free(iommu);

drivers/net/wireless/ath/ath11k/ahb.c

1077

struct iommu_domain *iommu;

drivers/net/wireless/ath/ath11k/ahb.c

1089

iommu = ab_ahb->fw.iommu_domain;

drivers/net/wireless/ath/ath11k/ahb.c

1091

unmapped_size = iommu_unmap(iommu, ab_ahb->fw.msa_paddr, ab_ahb->fw.msa_size);

drivers/net/wireless/ath/ath11k/ahb.c

1096

unmapped_size = iommu_unmap(iommu, ab_ahb->fw.ce_paddr, ab_ahb->fw.ce_size);

drivers/net/wireless/ath/ath11k/ahb.c

1101

iommu_detach_device(iommu, ab_ahb->fw.dev);

drivers/net/wireless/ath/ath11k/ahb.c

1102

iommu_domain_free(iommu);

drivers/parisc/ccio-dma.c

1557

hba->iommu = ioc;

drivers/parisc/dino.c

1003

dino_dev->hba.iommu = ccio_get_iommu(dev);

drivers/parisc/eisa.c

304

eisa_dev.hba.iommu = ccio_get_iommu(dev);

drivers/parisc/iommu.h

35

return pdata->iommu;

drivers/parisc/lba_pci.c

1562

lba_dev->hba.iommu = sba_get_iommu(dev); /* get iommu data */

drivers/vdpa/vdpa_sim/vdpa_sim.c

150

vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],

drivers/vdpa/vdpa_sim/vdpa_sim.c

156

vhost_iotlb_reset(&vdpasim->iommu[i]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

157

vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,

drivers/vdpa/vdpa_sim/vdpa_sim.c

253

vdpasim->iommu = kmalloc_objs(*vdpasim->iommu, vdpasim->dev_attr.nas);

drivers/vdpa/vdpa_sim/vdpa_sim.c

254

if (!vdpasim->iommu)

drivers/vdpa/vdpa_sim/vdpa_sim.c

263

vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);

drivers/vdpa/vdpa_sim/vdpa_sim.c

264

vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 0,

drivers/vdpa/vdpa_sim/vdpa_sim.c

270

vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],

drivers/vdpa/vdpa_sim/vdpa_sim.c

604

struct vhost_iotlb *iommu;

drivers/vdpa/vdpa_sim/vdpa_sim.c

607

iommu = &vdpasim->iommu[asid];

drivers/vdpa/vdpa_sim/vdpa_sim.c

613

vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,

drivers/vdpa/vdpa_sim/vdpa_sim.c

626

struct vhost_iotlb *iommu;

drivers/vdpa/vdpa_sim/vdpa_sim.c

635

iommu = &vdpasim->iommu[asid];

drivers/vdpa/vdpa_sim/vdpa_sim.c

636

vhost_iotlb_reset(iommu);

drivers/vdpa/vdpa_sim/vdpa_sim.c

641

ret = vhost_iotlb_add_range(iommu, map->start,

drivers/vdpa/vdpa_sim/vdpa_sim.c

650

vhost_iotlb_reset(iommu);

drivers/vdpa/vdpa_sim/vdpa_sim.c

665

vhost_iotlb_reset(&vdpasim->iommu[asid]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

666

vhost_iotlb_add_range(&vdpasim->iommu[asid], 0, ULONG_MAX,

drivers/vdpa/vdpa_sim/vdpa_sim.c

710

vhost_iotlb_reset(&vdpasim->iommu[asid]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

713

ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova,

drivers/vdpa/vdpa_sim/vdpa_sim.c

729

vhost_iotlb_reset(&vdpasim->iommu[asid]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

734

vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1);

drivers/vdpa/vdpa_sim/vdpa_sim.c

756

vhost_iotlb_reset(&vdpasim->iommu[i]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

757

kfree(vdpasim->iommu);

drivers/vdpa/vdpa_sim/vdpa_sim.h

68

struct vhost_iotlb *iommu;

drivers/vfio/vfio_iommu_type1.c

1000

iommu->num_non_pinned_groups--;

drivers/vfio/vfio_iommu_type1.c

1011

dma = vfio_find_dma(iommu, iova, PAGE_SIZE);

drivers/vfio/vfio_iommu_type1.c

1016

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1023

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

1030

if (WARN_ON(!iommu->v2))

drivers/vfio/vfio_iommu_type1.c

1040

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1042

do_accounting = list_empty(&iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

1047

dma = vfio_find_dma(iommu, iova, PAGE_SIZE);

drivers/vfio/vfio_iommu_type1.c

1054

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1148

static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,

drivers/vfio/vfio_iommu_type1.c

1161

if (list_empty(&iommu->domain_list))

drivers/vfio/vfio_iommu_type1.c

1171

domain = d = list_first_entry(&iommu->domain_list,

drivers/vfio/vfio_iommu_type1.c

1174

list_for_each_entry_continue(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

1234

static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)

drivers/vfio/vfio_iommu_type1.c

1237

vfio_unmap_unpin(iommu, dma, true);

drivers/vfio/vfio_iommu_type1.c

1238

vfio_unlink_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1243

iommu->vaddr_invalid_count--;

drivers/vfio/vfio_iommu_type1.c

1245

iommu->dma_avail++;

drivers/vfio/vfio_iommu_type1.c

1248

static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

1252

iommu->pgsize_bitmap = ULONG_MAX;

drivers/vfio/vfio_iommu_type1.c

1254

list_for_each_entry(domain, &iommu->domain_list, next)

drivers/vfio/vfio_iommu_type1.c

1255

iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap;

drivers/vfio/vfio_iommu_type1.c

1265

if (iommu->pgsize_bitmap & ~PAGE_MASK) {

drivers/vfio/vfio_iommu_type1.c

1266

iommu->pgsize_bitmap &= PAGE_MASK;

drivers/vfio/vfio_iommu_type1.c

1267

iommu->pgsize_bitmap |= PAGE_SIZE;

drivers/vfio/vfio_iommu_type1.c

1271

static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1286

if (iommu->num_non_pinned_groups && dma->iommu_mapped)

drivers/vfio/vfio_iommu_type1.c

1308

static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1322

dma = vfio_find_dma(iommu, iova, 1);

drivers/vfio/vfio_iommu_type1.c

1326

dma = vfio_find_dma(iommu, iova_end, 1);

drivers/vfio/vfio_iommu_type1.c

1330

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

1339

ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);

drivers/vfio/vfio_iommu_type1.c

1368

static void vfio_notify_dma_unmap(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1373

if (list_empty(&iommu->device_list))

drivers/vfio/vfio_iommu_type1.c

1382

mutex_lock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

1383

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1385

list_for_each_entry(device, &iommu->device_list, iommu_entry)

drivers/vfio/vfio_iommu_type1.c

1388

mutex_unlock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

1389

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1392

static int vfio_dma_do_unmap(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1407

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1410

if (invalidate_vaddr && !list_empty(&iommu->emulated_iommu_groups)) {

drivers/vfio/vfio_iommu_type1.c

1420

pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

1442

(!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) {

drivers/vfio/vfio_iommu_type1.c

1479

if (iommu->v2 && !unmap_all) {

drivers/vfio/vfio_iommu_type1.c

1480

dma = vfio_find_dma(iommu, iova, 1);

drivers/vfio/vfio_iommu_type1.c

1484

dma = vfio_find_dma(iommu, iova_end, 1);

drivers/vfio/vfio_iommu_type1.c

1490

n = first_n = vfio_find_dma_first_node(iommu, iova, iova_end);

drivers/vfio/vfio_iommu_type1.c

1497

if (!iommu->v2 && iova > dma->iova)

drivers/vfio/vfio_iommu_type1.c

1508

iommu->vaddr_invalid_count--;

drivers/vfio/vfio_iommu_type1.c

1515

iommu->vaddr_invalid_count++;

drivers/vfio/vfio_iommu_type1.c

1529

vfio_notify_dma_unmap(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1534

ret = update_user_bitmap(bitmap->data, iommu, dma,

drivers/vfio/vfio_iommu_type1.c

1542

vfio_remove_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1546

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1554

static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova,

drivers/vfio/vfio_iommu_type1.c

1560

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

1573

list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

158

vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1581

static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,

drivers/vfio/vfio_iommu_type1.c

1606

ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,

drivers/vfio/vfio_iommu_type1.c

1623

vfio_remove_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1631

static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1634

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

166

static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1681

static int vfio_dma_do_map(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

169

struct rb_node *node = iommu->dma_list.rb_node;

drivers/vfio/vfio_iommu_type1.c

1714

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1716

pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

1725

dma = vfio_find_dma(iommu, iova, size);

drivers/vfio/vfio_iommu_type1.c

1738

iommu->vaddr_invalid_count--;

drivers/vfio/vfio_iommu_type1.c

1746

if (!iommu->dma_avail) {

drivers/vfio/vfio_iommu_type1.c

1751

if (!vfio_iommu_iova_dma_valid(iommu, iova, iova_end)) {

drivers/vfio/vfio_iommu_type1.c

1762

iommu->dma_avail--;

drivers/vfio/vfio_iommu_type1.c

1785

vfio_link_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1788

if (list_empty(&iommu->domain_list))

drivers/vfio/vfio_iommu_type1.c

1791

ret = vfio_pin_map_dma(iommu, dma, size);

drivers/vfio/vfio_iommu_type1.c

1793

if (!ret && iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

1796

vfio_remove_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1800

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1804

static int vfio_iommu_replay(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1814

if (!list_empty(&iommu->domain_list))

drivers/vfio/vfio_iommu_type1.c

1815

d = list_first_entry(&iommu->domain_list,

drivers/vfio/vfio_iommu_type1.c

1820

n = rb_first(&iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

187

static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1897

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

192

struct rb_node *node = iommu->dma_list.rb_node;

drivers/vfio/vfio_iommu_type1.c

1962

vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1968

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

1974

list_for_each_entry(group, &iommu->emulated_iommu_groups, next)

drivers/vfio/vfio_iommu_type1.c

2036

static bool vfio_iommu_aper_conflict(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2040

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

2053

if (vfio_find_dma(iommu, first->start, start - first->start))

drivers/vfio/vfio_iommu_type1.c

2059

if (vfio_find_dma(iommu, end + 1, last->end - end))

drivers/vfio/vfio_iommu_type1.c

2110

static bool vfio_iommu_resv_conflict(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2120

if (vfio_find_dma(iommu, region->start, region->length))

drivers/vfio/vfio_iommu_type1.c

215

static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new)

drivers/vfio/vfio_iommu_type1.c

217

struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL;

drivers/vfio/vfio_iommu_type1.c

2199

static int vfio_iommu_iova_get_copy(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2202

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

2219

static void vfio_iommu_iova_insert_copy(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2222

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

2240

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

2250

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2253

if (iommu->vaddr_invalid_count)

drivers/vfio/vfio_iommu_type1.c

2258

if (vfio_iommu_find_iommu_group(iommu, iommu_group))

drivers/vfio/vfio_iommu_type1.c

2268

list_add(&group->next, &iommu->emulated_iommu_groups);

drivers/vfio/vfio_iommu_type1.c

2303

if (vfio_iommu_aper_conflict(iommu, geo->aperture_start,

drivers/vfio/vfio_iommu_type1.c

2313

if (vfio_iommu_resv_conflict(iommu, &group_resv_regions)) {

drivers/vfio/vfio_iommu_type1.c

2323

ret = vfio_iommu_iova_get_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

233

rb_insert_color(&new->node, &iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

236

static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)

drivers/vfio/vfio_iommu_type1.c

2366

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

238

rb_erase(&old->node, &iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

2387

ret = vfio_iommu_replay(iommu, domain);

drivers/vfio/vfio_iommu_type1.c

2397

list_add(&domain->next, &iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

2398

vfio_update_pgsize_bitmap(iommu);

drivers/vfio/vfio_iommu_type1.c

2401

vfio_iommu_iova_insert_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2408

iommu->num_non_pinned_groups++;

drivers/vfio/vfio_iommu_type1.c

2409

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2425

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2429

static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2433

while ((node = rb_first(&iommu->dma_list)))

drivers/vfio/vfio_iommu_type1.c

2434

vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));

drivers/vfio/vfio_iommu_type1.c

2437

static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2441

n = rb_first(&iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

2447

unlocked += vfio_unmap_unpin(iommu, dma, false);

drivers/vfio/vfio_iommu_type1.c

2465

static void vfio_iommu_aper_expand(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2476

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2498

static int vfio_iommu_resv_refresh(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2511

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2542

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

2548

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2549

list_for_each_entry(group, &iommu->emulated_iommu_groups, next) {

drivers/vfio/vfio_iommu_type1.c

2556

if (list_empty(&iommu->emulated_iommu_groups) &&

drivers/vfio/vfio_iommu_type1.c

2557

list_empty(&iommu->domain_list)) {

drivers/vfio/vfio_iommu_type1.c

2558

WARN_ON(!list_empty(&iommu->device_list));

drivers/vfio/vfio_iommu_type1.c

2559

vfio_iommu_unmap_unpin_all(iommu);

drivers/vfio/vfio_iommu_type1.c

2569

vfio_iommu_iova_get_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2571

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2588

if (list_is_singular(&iommu->domain_list)) {

drivers/vfio/vfio_iommu_type1.c

2589

if (list_empty(&iommu->emulated_iommu_groups)) {

drivers/vfio/vfio_iommu_type1.c

2591

&iommu->device_list));

drivers/vfio/vfio_iommu_type1.c

2592

vfio_iommu_unmap_unpin_all(iommu);

drivers/vfio/vfio_iommu_type1.c

2594

vfio_iommu_unmap_unpin_reaccount(iommu);

drivers/vfio/vfio_iommu_type1.c

2600

vfio_iommu_aper_expand(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2601

vfio_update_pgsize_bitmap(iommu);

drivers/vfio/vfio_iommu_type1.c

2606

if (!vfio_iommu_resv_refresh(iommu, &iova_copy))

drivers/vfio/vfio_iommu_type1.c

2607

vfio_iommu_iova_insert_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2617

iommu->num_non_pinned_groups--;

drivers/vfio/vfio_iommu_type1.c

2618

if (iommu->dirty_page_tracking)

drivers/vfio/vfio_iommu_type1.c

2619

vfio_iommu_populate_bitmap_full(iommu);

drivers/vfio/vfio_iommu_type1.c

2621

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2626

struct vfio_iommu *iommu;

drivers/vfio/vfio_iommu_type1.c

2628

iommu = kzalloc_obj(*iommu);

drivers/vfio/vfio_iommu_type1.c

2629

if (!iommu)

drivers/vfio/vfio_iommu_type1.c

2637

iommu->v2 = true;

drivers/vfio/vfio_iommu_type1.c

2640

kfree(iommu);

drivers/vfio/vfio_iommu_type1.c

2644

INIT_LIST_HEAD(&iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

2645

INIT_LIST_HEAD(&iommu->iova_list);

drivers/vfio/vfio_iommu_type1.c

2646

iommu->dma_list = RB_ROOT;

drivers/vfio/vfio_iommu_type1.c

2647

iommu->dma_avail = dma_entry_limit;

drivers/vfio/vfio_iommu_type1.c

2648

mutex_init(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2649

mutex_init(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

2650

INIT_LIST_HEAD(&iommu->device_list);

drivers/vfio/vfio_iommu_type1.c

2651

iommu->pgsize_bitmap = PAGE_MASK;

drivers/vfio/vfio_iommu_type1.c

2652

INIT_LIST_HEAD(&iommu->emulated_iommu_groups);

drivers/vfio/vfio_iommu_type1.c

2654

return iommu;

drivers/vfio/vfio_iommu_type1.c

2673

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

2678

&iommu->emulated_iommu_groups, next) {

drivers/vfio/vfio_iommu_type1.c

2683

vfio_iommu_unmap_unpin_all(iommu);

drivers/vfio/vfio_iommu_type1.c

2686

&iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2692

vfio_iommu_iova_free(&iommu->iova_list);

drivers/vfio/vfio_iommu_type1.c

2694

kfree(iommu);

drivers/vfio/vfio_iommu_type1.c

2697

static int vfio_domains_have_enforce_cache_coherency(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2702

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2703

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2709

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2714

static bool vfio_iommu_has_emulated(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2718

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2719

ret = !list_empty(&iommu->emulated_iommu_groups);

drivers/vfio/vfio_iommu_type1.c

2720

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2724

static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2737

return iommu && !vfio_iommu_has_emulated(iommu);

drivers/vfio/vfio_iommu_type1.c

2739

if (!iommu)

drivers/vfio/vfio_iommu_type1.c

2741

return vfio_domains_have_enforce_cache_coherency(iommu);

drivers/vfio/vfio_iommu_type1.c

2768

static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2776

list_for_each_entry(iova, &iommu->iova_list, list)

drivers/vfio/vfio_iommu_type1.c

2795

list_for_each_entry(iova, &iommu->iova_list, list) {

drivers/vfio/vfio_iommu_type1.c

280

static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2807

static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2817

cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

2823

static int vfio_iommu_dma_avail_build_caps(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

283

unsigned long pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

2831

cap_dma_avail.avail = iommu->dma_avail;

drivers/vfio/vfio_iommu_type1.c

2837

static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

285

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

2855

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2858

info.iova_pgsizes = iommu->pgsize_bitmap;

drivers/vfio/vfio_iommu_type1.c

2860

ret = vfio_iommu_migration_build_caps(iommu, &caps);

drivers/vfio/vfio_iommu_type1.c

2863

ret = vfio_iommu_dma_avail_build_caps(iommu, &caps);

drivers/vfio/vfio_iommu_type1.c

2866

ret = vfio_iommu_iova_build_caps(iommu, &caps);

drivers/vfio/vfio_iommu_type1.c

2868

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2896

static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2912

return vfio_dma_do_map(iommu, &map);

drivers/vfio/vfio_iommu_type1.c

2915

static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

292

static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize)

drivers/vfio/vfio_iommu_type1.c

296

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

2960

ret = vfio_dma_do_unmap(iommu, &unmap, &bitmap);

drivers/vfio/vfio_iommu_type1.c

2968

static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2978

if (!iommu->v2)

drivers/vfio/vfio_iommu_type1.c

2996

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2997

pgsize = 1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

2998

if (!iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

2999

ret = vfio_dma_bitmap_alloc_all(iommu, pgsize);

drivers/vfio/vfio_iommu_type1.c

3001

iommu->dirty_page_tracking = true;

drivers/vfio/vfio_iommu_type1.c

3003

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3006

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3007

if (iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

3008

iommu->dirty_page_tracking = false;

drivers/vfio/vfio_iommu_type1.c

3009

vfio_dma_bitmap_free_all(iommu);

drivers/vfio/vfio_iommu_type1.c

3011

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3049

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3051

iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

3067

if (iommu->dirty_page_tracking)

drivers/vfio/vfio_iommu_type1.c

3069

iommu, iova, iova_end,

drivers/vfio/vfio_iommu_type1.c

3074

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3085

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3089

return vfio_iommu_type1_check_extension(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3091

return vfio_iommu_type1_get_info(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3093

return vfio_iommu_type1_map_dma(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3095

return vfio_iommu_type1_unmap_dma(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3097

return vfio_iommu_type1_dirty_pages(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3106

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3117

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3118

mutex_lock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3119

list_add(&vdev->iommu_entry, &iommu->device_list);

drivers/vfio/vfio_iommu_type1.c

3120

mutex_unlock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3121

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3127

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3132

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3133

mutex_lock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3135

mutex_unlock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3136

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3139

static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

3152

dma = vfio_find_dma(iommu, user_iova, 1);

drivers/vfio/vfio_iommu_type1.c

317

static void vfio_dma_bitmap_free_all(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

3179

if (*copied && iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

3180

unsigned long pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

3202

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3206

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3208

if (WARN_ONCE(iommu->vaddr_invalid_count,

drivers/vfio/vfio_iommu_type1.c

321

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

3215

ret = vfio_iommu_type1_dma_rw_chunk(iommu, user_iova, data,

drivers/vfio/vfio_iommu_type1.c

3226

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3235

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3238

if (!iommu || !iommu_group)

drivers/vfio/vfio_iommu_type1.c

3241

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3242

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

3248

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

900

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

909

if (!iommu || !pages || npage <= 0)

drivers/vfio/vfio_iommu_type1.c

913

if (!iommu->v2)

drivers/vfio/vfio_iommu_type1.c

920

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

922

if (WARN_ONCE(iommu->vaddr_invalid_count,

drivers/vfio/vfio_iommu_type1.c

929

if (list_empty(&iommu->device_list)) {

drivers/vfio/vfio_iommu_type1.c

939

do_accounting = list_empty(&iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

947

dma = vfio_find_dma(iommu, iova, PAGE_SIZE);

drivers/vfio/vfio_iommu_type1.c

984

if (iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

985

unsigned long pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

997

group = vfio_iommu_find_iommu_group(iommu, iommu_group);

drivers/xen/grant-dma-iommu.c

16

struct iommu_device iommu;

drivers/xen/grant-dma-iommu.c

45

ret = iommu_device_register(&mmu->iommu, &grant_dma_iommu_ops, &pdev->dev);

drivers/xen/grant-dma-iommu.c

59

iommu_device_unregister(&mmu->iommu);

include/linux/amd-iommu.h

65

int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn,

include/linux/amd-iommu.h

67

int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn,

include/linux/device.h

653

struct dev_iommu *iommu;

include/linux/dmar.h

130

void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,

include/linux/dmar.h

133

static inline void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,

include/linux/dmar.h

296

extern int dmar_set_interrupt(struct intel_iommu *iommu);

include/linux/dmar.h

49

struct intel_iommu *iommu;

include/linux/dmar.h

82

if (i=drhd->iommu, drhd->ignored) {} else

include/linux/dmar.h

87

if (i=drhd->iommu, 0) {} else

include/linux/generic_pt/iommu.h

215

struct pt_iommu iommu; \

include/linux/iommu.h

1150

if (dev->iommu)

include/linux/iommu.h

1151

return dev->iommu->fwspec;

include/linux/iommu.h

1159

dev->iommu->fwspec = fwspec;

include/linux/iommu.h

1164

if (dev->iommu)

include/linux/iommu.h

1165

return dev->iommu->priv;

include/linux/iommu.h

1387

static inline int iommu_device_register(struct iommu_device *iommu,

include/linux/iommu.h

1426

static inline void iommu_device_unregister(struct iommu_device *iommu)

include/linux/iommu.h

1430

static inline int iommu_device_sysfs_add(struct iommu_device *iommu,

include/linux/iommu.h

1438

static inline void iommu_device_sysfs_remove(struct iommu_device *iommu)

include/linux/iommu.h

858

int iommu_device_register(struct iommu_device *iommu,

include/linux/iommu.h

861

void iommu_device_unregister(struct iommu_device *iommu);

include/linux/iommu.h

862

int iommu_device_sysfs_add(struct iommu_device *iommu,

include/linux/iommu.h

866

void iommu_device_sysfs_remove(struct iommu_device *iommu);

include/linux/iommu.h

867

int iommu_device_link(struct iommu_device *iommu, struct device *link);

include/linux/iommu.h

868

void iommu_device_unlink(struct iommu_device *iommu, struct device *link);

include/linux/iommu.h

886

return dev->iommu->iommu_dev;

include/trace/events/iommu.h

9

#define TRACE_SYSTEM iommu

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

35

struct iommu *iommu_init(const char *iommu_mode);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

36

void iommu_cleanup(struct iommu *iommu);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

38

int __iommu_map(struct iommu *iommu, struct dma_region *region);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

40

static inline void iommu_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

42

VFIO_ASSERT_EQ(__iommu_map(iommu, region), 0);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

45

int __iommu_unmap(struct iommu *iommu, struct dma_region *region, u64 *unmapped);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

47

static inline void iommu_unmap(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

49

VFIO_ASSERT_EQ(__iommu_unmap(iommu, region, NULL), 0);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

52

int __iommu_unmap_all(struct iommu *iommu, u64 *unmapped);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

54

static inline void iommu_unmap_all(struct iommu *iommu)

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

56

VFIO_ASSERT_EQ(__iommu_unmap_all(iommu, NULL), 0);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

59

int __iommu_hva2iova(struct iommu *iommu, void *vaddr, iova_t *iova);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

60

iova_t iommu_hva2iova(struct iommu *iommu, void *vaddr);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

62

struct iommu_iova_range *iommu_iova_ranges(struct iommu *iommu, u32 *nranges);

tools/testing/selftests/vfio/lib/include/libvfio/iova_allocator.h

18

struct iova_allocator *iova_allocator_init(struct iommu *iommu);

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

108

return __iommu_hva2iova(device->iommu, vaddr, iova);

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

113

return iommu_hva2iova(device->iommu, vaddr);

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

23

struct iommu *iommu;

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

41

struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu);

tools/testing/selftests/vfio/lib/iommu.c

100

static int vfio_iommu_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/iommu.c

110

if (ioctl(iommu->container_fd, VFIO_IOMMU_MAP_DMA, &args))

tools/testing/selftests/vfio/lib/iommu.c

116

static int iommufd_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/iommu.c

126

.ioas_id = iommu->ioas_id,

tools/testing/selftests/vfio/lib/iommu.c

129

if (ioctl(iommu->iommufd, IOMMU_IOAS_MAP, &args))

tools/testing/selftests/vfio/lib/iommu.c

135

int __iommu_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/iommu.c

139

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

140

ret = iommufd_map(iommu, region);

tools/testing/selftests/vfio/lib/iommu.c

142

ret = vfio_iommu_map(iommu, region);

tools/testing/selftests/vfio/lib/iommu.c

147

list_add(&region->link, &iommu->dma_regions);

tools/testing/selftests/vfio/lib/iommu.c

170

static int vfio_iommu_unmap(struct iommu *iommu, struct dma_region *region,

tools/testing/selftests/vfio/lib/iommu.c

173

return __vfio_iommu_unmap(iommu->container_fd, region->iova,

tools/testing/selftests/vfio/lib/iommu.c

195

static int iommufd_unmap(struct iommu *iommu, struct dma_region *region,

tools/testing/selftests/vfio/lib/iommu.c

198

return __iommufd_unmap(iommu->iommufd, region->iova, region->size,

tools/testing/selftests/vfio/lib/iommu.c

199

iommu->ioas_id, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

202

int __iommu_unmap(struct iommu *iommu, struct dma_region *region, u64 *unmapped)

tools/testing/selftests/vfio/lib/iommu.c

206

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

207

ret = iommufd_unmap(iommu, region, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

209

ret = vfio_iommu_unmap(iommu, region, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

219

int __iommu_unmap_all(struct iommu *iommu, u64 *unmapped)

tools/testing/selftests/vfio/lib/iommu.c

224

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

225

ret = __iommufd_unmap(iommu->iommufd, 0, UINT64_MAX,

tools/testing/selftests/vfio/lib/iommu.c

226

iommu->ioas_id, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

228

ret = __vfio_iommu_unmap(iommu->container_fd, 0, 0,

tools/testing/selftests/vfio/lib/iommu.c

234

list_for_each_entry_safe(curr, next, &iommu->dma_regions, link)

tools/testing/selftests/vfio/lib/iommu.c

312

static struct iommu_iova_range *vfio_iommu_iova_ranges(struct iommu *iommu,

tools/testing/selftests/vfio/lib/iommu.c

320

info = vfio_iommu_get_info(iommu->container_fd);

tools/testing/selftests/vfio/lib/iommu.c

344

static struct iommu_iova_range *iommufd_iova_ranges(struct iommu *iommu,

tools/testing/selftests/vfio/lib/iommu.c

352

.ioas_id = iommu->ioas_id,

tools/testing/selftests/vfio/lib/iommu.c

355

ret = ioctl(iommu->iommufd, IOMMU_IOAS_IOVA_RANGES, &query);

tools/testing/selftests/vfio/lib/iommu.c

365

ioctl_assert(iommu->iommufd, IOMMU_IOAS_IOVA_RANGES, &query);

tools/testing/selftests/vfio/lib/iommu.c

385

struct iommu_iova_range *iommu_iova_ranges(struct iommu *iommu, u32 *nranges)

tools/testing/selftests/vfio/lib/iommu.c

389

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

390

ranges = iommufd_iova_ranges(iommu, nranges);

tools/testing/selftests/vfio/lib/iommu.c

392

ranges = vfio_iommu_iova_ranges(iommu, nranges);

tools/testing/selftests/vfio/lib/iommu.c

421

struct iommu *iommu_init(const char *iommu_mode)

tools/testing/selftests/vfio/lib/iommu.c

424

struct iommu *iommu;

tools/testing/selftests/vfio/lib/iommu.c

427

iommu = calloc(1, sizeof(*iommu));

tools/testing/selftests/vfio/lib/iommu.c

428

VFIO_ASSERT_NOT_NULL(iommu);

tools/testing/selftests/vfio/lib/iommu.c

430

INIT_LIST_HEAD(&iommu->dma_regions);

tools/testing/selftests/vfio/lib/iommu.c

432

iommu->mode = lookup_iommu_mode(iommu_mode);

tools/testing/selftests/vfio/lib/iommu.c

434

container_path = iommu->mode->container_path;

tools/testing/selftests/vfio/lib/iommu.c

436

iommu->container_fd = open(container_path, O_RDWR);

tools/testing/selftests/vfio/lib/iommu.c

437

VFIO_ASSERT_GE(iommu->container_fd, 0, "open(%s) failed\n", container_path);

tools/testing/selftests/vfio/lib/iommu.c

439

version = ioctl(iommu->container_fd, VFIO_GET_API_VERSION);

tools/testing/selftests/vfio/lib/iommu.c

447

iommu->iommufd = open("/dev/iommu", O_RDWR);

tools/testing/selftests/vfio/lib/iommu.c

448

VFIO_ASSERT_GT(iommu->iommufd, 0);

tools/testing/selftests/vfio/lib/iommu.c

450

iommu->ioas_id = iommufd_ioas_alloc(iommu->iommufd);

tools/testing/selftests/vfio/lib/iommu.c

453

return iommu;

tools/testing/selftests/vfio/lib/iommu.c

456

void iommu_cleanup(struct iommu *iommu)

tools/testing/selftests/vfio/lib/iommu.c

458

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

459

VFIO_ASSERT_EQ(close(iommu->iommufd), 0);

tools/testing/selftests/vfio/lib/iommu.c

461

VFIO_ASSERT_EQ(close(iommu->container_fd), 0);

tools/testing/selftests/vfio/lib/iommu.c

463

free(iommu);

tools/testing/selftests/vfio/lib/iommu.c

69

int __iommu_hva2iova(struct iommu *iommu, void *vaddr, iova_t *iova)

tools/testing/selftests/vfio/lib/iommu.c

73

list_for_each_entry(region, &iommu->dma_regions, link) {

tools/testing/selftests/vfio/lib/iommu.c

89

iova_t iommu_hva2iova(struct iommu *iommu, void *vaddr)

tools/testing/selftests/vfio/lib/iommu.c

94

ret = __iommu_hva2iova(iommu, vaddr, &iova);

tools/testing/selftests/vfio/lib/iova_allocator.c

23

struct iova_allocator *iova_allocator_init(struct iommu *iommu)

tools/testing/selftests/vfio/lib/iova_allocator.c

29

ranges = iommu_iova_ranges(iommu, &nranges);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

240

ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->iommu->container_fd);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

245

struct iommu *iommu = device->iommu;

tools/testing/selftests/vfio/lib/vfio_pci_device.c

246

unsigned long iommu_type = iommu->mode->iommu_type;

tools/testing/selftests/vfio/lib/vfio_pci_device.c

251

ret = ioctl(iommu->container_fd, VFIO_CHECK_EXTENSION, iommu_type);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

259

(void)ioctl(iommu->container_fd, VFIO_SET_IOMMU, (void *)iommu_type);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

351

vfio_device_bind_iommufd(device->fd, device->iommu->iommufd);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

352

vfio_device_attach_iommufd_pt(device->fd, device->iommu->ioas_id);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

355

struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu)

tools/testing/selftests/vfio/lib/vfio_pci_device.c

362

VFIO_ASSERT_NOT_NULL(iommu);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

363

device->iommu = iommu;

tools/testing/selftests/vfio/lib/vfio_pci_device.c

366

if (iommu->mode->container_path)

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

100

iommu_unmap(iommu, &region);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

102

VFIO_ASSERT_NE(__iommu_map(iommu, &region), 0);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

103

VFIO_ASSERT_NE(__iommu_unmap(iommu, &region, NULL), 0);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

109

do_mmio_map_test(self->iommu, self->iova_allocator,

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

118

do_mmio_map_test(self->iommu, self->iova_allocator,

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

134

do_mmio_map_test(self->iommu, self->iova_allocator, vaddr, size);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

46

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

67

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

68

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

69

self->iova_allocator = iova_allocator_init(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

80

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

83

static void do_mmio_map_test(struct iommu *iommu,

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

97

if (!strcmp(iommu->mode->name, MODE_VFIO_TYPE1V2_IOMMU) ||

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

98

!strcmp(iommu->mode->name, MODE_VFIO_TYPE1_IOMMU)) {

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

99

iommu_map(iommu, &region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

122

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

123

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

124

self->iova_allocator = iova_allocator_init(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

131

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

155

iommu_map(self->iommu, &region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

164

if (self->iommu->mode->iommu_type == VFIO_TYPE1_IOMMU)

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

193

rc = __iommu_unmap(self->iommu, &region, &unmapped);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

204

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

237

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

238

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

243

ranges = iommu_iova_ranges(self->iommu, &nranges);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

256

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

266

iommu_map(self->iommu, region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

269

rc = __iommu_unmap(self->iommu, region, &unmapped);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

280

iommu_map(self->iommu, region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

283

rc = __iommu_unmap_all(self->iommu, &unmapped);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

296

rc = __iommu_map(self->iommu, region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

299

rc = __iommu_unmap(self->iommu, region, NULL);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

96

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

105

device = vfio_pci_device_init(device_bdfs[args->device_index], args->iommu);

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

18

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

29

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

47

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

54

self->thread_args[i].iommu = self->iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

62

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

105

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_test.c

123

self->iommu = iommu_init(default_iommu_mode);

tools/testing/selftests/vfio/vfio_pci_device_test.c

124

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

130

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

26

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_test.c

32

self->iommu = iommu_init(default_iommu_mode);

tools/testing/selftests/vfio/vfio_pci_device_test.c

33

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

39

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

112

region_teardown(self->iommu, &self->memcpy_region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

113

region_teardown(self->iommu, &driver->region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

117

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

21

static void region_setup(struct iommu *iommu,

tools/testing/selftests/vfio/vfio_pci_driver_test.c

239

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_driver_test.c

242

iommu = iommu_init(default_iommu_mode);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

243

device = vfio_pci_device_init(device_bdf, iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

248

iommu_cleanup(iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

36

iommu_map(iommu, region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

39

static void region_teardown(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/vfio_pci_driver_test.c

41

iommu_unmap(iommu, region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

46

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_driver_test.c

76

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

77

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

78

self->iova_allocator = iova_allocator_init(self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

82

region_setup(self->iommu, self->iova_allocator, &self->memcpy_region, SZ_1G);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

83

region_setup(self->iommu, self->iova_allocator, &driver->region, SZ_2M);