arch/parisc/include/asm/pci.h

59

struct ioc *iommu; /* IOMMU this device is under */

arch/powerpc/include/asm/pci-bridge.h

138

struct iommu_device iommu;

arch/powerpc/kernel/iommu.c

1243

return &hose->iommu;

arch/powerpc/kernel/iommu.c

1289

iommu_device_sysfs_add(&phb->iommu, phb->parent,

arch/powerpc/kernel/iommu.c

1292

iommu_device_register(&phb->iommu, &spapr_tce_iommu_ops,

arch/powerpc/kernel/iommu.c

1298

iommu_device_unregister(&phb->iommu);

arch/powerpc/kernel/iommu.c

1299

iommu_device_sysfs_remove(&phb->iommu);

arch/sparc/include/asm/device.h

14

void *iommu;

arch/sparc/include/asm/iommu-common.h

35

extern void iommu_tbl_pool_init(struct iommu_map_table *iommu,

arch/sparc/include/asm/iommu-common.h

43

struct iommu_map_table *iommu,

arch/sparc/include/asm/iommu-common.h

49

extern void iommu_tbl_range_free(struct iommu_map_table *iommu,

arch/sparc/include/asm/iommu_64.h

89

int iommu_table_init(struct iommu *iommu, int tsbsize,

arch/sparc/kernel/iommu-common.c

109

unsigned int npools = iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

111

bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0);

arch/sparc/kernel/iommu-common.c

126

pool = &(iommu->large_pool);

arch/sparc/kernel/iommu-common.c

131

pool = &(iommu->pools[pool_nr]);

arch/sparc/kernel/iommu-common.c

152

shift = iommu->table_map_base >> iommu->table_shift;

arch/sparc/kernel/iommu-common.c

161

pool = &(iommu->pools[0]);

arch/sparc/kernel/iommu-common.c

174

if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) {

arch/sparc/kernel/iommu-common.c

176

boundary_size = iommu->poolsize * iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

179

iommu->table_shift);

arch/sparc/kernel/iommu-common.c

181

n = iommu_area_alloc(iommu->map, limit, start, npages, shift,

arch/sparc/kernel/iommu-common.c

187

set_flush(iommu);

arch/sparc/kernel/iommu-common.c

19

static inline bool need_flush(struct iommu_map_table *iommu)

arch/sparc/kernel/iommu-common.c

190

} else if (!largealloc && pass <= iommu->nr_pools) {

arch/sparc/kernel/iommu-common.c

192

pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);

arch/sparc/kernel/iommu-common.c

193

pool = &(iommu->pools[pool_nr]);

arch/sparc/kernel/iommu-common.c

196

set_flush(iommu);

arch/sparc/kernel/iommu-common.c

205

if (iommu->lazy_flush &&

arch/sparc/kernel/iommu-common.c

206

(n < pool->hint || need_flush(iommu))) {

arch/sparc/kernel/iommu-common.c

207

clear_flush(iommu);

arch/sparc/kernel/iommu-common.c

208

iommu->lazy_flush(iommu);

arch/sparc/kernel/iommu-common.c

21

return ((iommu->flags & IOMMU_NEED_FLUSH) != 0);

arch/sparc/kernel/iommu-common.c

24

static inline void set_flush(struct iommu_map_table *iommu)

arch/sparc/kernel/iommu-common.c

246

void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr,

arch/sparc/kernel/iommu-common.c

251

unsigned long shift = iommu->table_shift;

arch/sparc/kernel/iommu-common.c

254

entry = (dma_addr - iommu->table_map_base) >> shift;

arch/sparc/kernel/iommu-common.c

255

pool = get_pool(iommu, entry);

arch/sparc/kernel/iommu-common.c

258

bitmap_clear(iommu->map, entry, npages);

arch/sparc/kernel/iommu-common.c

26

iommu->flags |= IOMMU_NEED_FLUSH;

arch/sparc/kernel/iommu-common.c

29

static inline void clear_flush(struct iommu_map_table *iommu)

arch/sparc/kernel/iommu-common.c

31

iommu->flags &= ~IOMMU_NEED_FLUSH;

arch/sparc/kernel/iommu-common.c

52

void iommu_tbl_pool_init(struct iommu_map_table *iommu,

arch/sparc/kernel/iommu-common.c

60

struct iommu_pool *p = &(iommu->large_pool);

arch/sparc/kernel/iommu-common.c

64

iommu->nr_pools = IOMMU_NR_POOLS;

arch/sparc/kernel/iommu-common.c

66

iommu->nr_pools = npools;

arch/sparc/kernel/iommu-common.c

69

iommu->table_shift = table_shift;

arch/sparc/kernel/iommu-common.c

70

iommu->lazy_flush = lazy_flush;

arch/sparc/kernel/iommu-common.c

73

iommu->flags |= IOMMU_NO_SPAN_BOUND;

arch/sparc/kernel/iommu-common.c

75

iommu->flags |= IOMMU_HAS_LARGE_POOL;

arch/sparc/kernel/iommu-common.c

78

iommu->poolsize = num_entries/iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

80

iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools;

arch/sparc/kernel/iommu-common.c

81

for (i = 0; i < iommu->nr_pools; i++) {

arch/sparc/kernel/iommu-common.c

82

spin_lock_init(&(iommu->pools[i].lock));

arch/sparc/kernel/iommu-common.c

83

iommu->pools[i].start = start;

arch/sparc/kernel/iommu-common.c

84

iommu->pools[i].hint = start;

arch/sparc/kernel/iommu-common.c

85

start += iommu->poolsize; /* start for next pool */

arch/sparc/kernel/iommu-common.c

86

iommu->pools[i].end = start - 1;

arch/sparc/kernel/iommu-common.c

98

struct iommu_map_table *iommu,

arch/sparc/kernel/iommu.c

103

spin_lock_init(&iommu->lock);

arch/sparc/kernel/iommu.c

104

iommu->ctx_lowest_free = 1;

arch/sparc/kernel/iommu.c

105

iommu->tbl.table_map_base = dma_offset;

arch/sparc/kernel/iommu.c

106

iommu->dma_addr_mask = dma_addr_mask;

arch/sparc/kernel/iommu.c

111

iommu->tbl.map = kzalloc_node(sz, GFP_KERNEL, numa_node);

arch/sparc/kernel/iommu.c

112

if (!iommu->tbl.map)

arch/sparc/kernel/iommu.c

115

iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,

arch/sparc/kernel/iommu.c

127

iommu->dummy_page = (unsigned long) page_address(page);

arch/sparc/kernel/iommu.c

128

memset((void *)iommu->dummy_page, 0, PAGE_SIZE);

arch/sparc/kernel/iommu.c

129

iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);

arch/sparc/kernel/iommu.c

138

iommu->page_table = (iopte_t *)page_address(page);

arch/sparc/kernel/iommu.c

141

iopte_make_dummy(iommu, &iommu->page_table[i]);

arch/sparc/kernel/iommu.c

146

free_page(iommu->dummy_page);

arch/sparc/kernel/iommu.c

147

iommu->dummy_page = 0UL;

arch/sparc/kernel/iommu.c

150

kfree(iommu->tbl.map);

arch/sparc/kernel/iommu.c

151

iommu->tbl.map = NULL;

arch/sparc/kernel/iommu.c

157

struct iommu *iommu,

arch/sparc/kernel/iommu.c

162

entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,

arch/sparc/kernel/iommu.c

167

return iommu->page_table + entry;

arch/sparc/kernel/iommu.c

170

static int iommu_alloc_ctx(struct iommu *iommu)

arch/sparc/kernel/iommu.c

172

int lowest = iommu->ctx_lowest_free;

arch/sparc/kernel/iommu.c

173

int n = find_next_zero_bit(iommu->ctx_bitmap, IOMMU_NUM_CTXS, lowest);

arch/sparc/kernel/iommu.c

176

n = find_next_zero_bit(iommu->ctx_bitmap, lowest, 1);

arch/sparc/kernel/iommu.c

183

__set_bit(n, iommu->ctx_bitmap);

arch/sparc/kernel/iommu.c

188

static inline void iommu_free_ctx(struct iommu *iommu, int ctx)

arch/sparc/kernel/iommu.c

191

__clear_bit(ctx, iommu->ctx_bitmap);

arch/sparc/kernel/iommu.c

192

if (ctx < iommu->ctx_lowest_free)

arch/sparc/kernel/iommu.c

193

iommu->ctx_lowest_free = ctx;

arch/sparc/kernel/iommu.c

202

struct iommu *iommu;

arch/sparc/kernel/iommu.c

221

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

223

iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT);

arch/sparc/kernel/iommu.c

230

*dma_addrp = (iommu->tbl.table_map_base +

arch/sparc/kernel/iommu.c

231

((iopte - iommu->page_table) << IO_PAGE_SHIFT));

arch/sparc/kernel/iommu.c

250

struct iommu *iommu;

arch/sparc/kernel/iommu.c

254

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

256

iommu_tbl_range_free(&iommu->tbl, dvma, npages, IOMMU_ERROR_CODE);

arch/sparc/kernel/iommu.c

267

struct iommu *iommu;

arch/sparc/kernel/iommu.c

285

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

295

base = alloc_npages(dev, iommu, npages);

arch/sparc/kernel/iommu.c

296

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

298

if (iommu->iommu_ctxflush)

arch/sparc/kernel/iommu.c

299

ctx = iommu_alloc_ctx(iommu);

arch/sparc/kernel/iommu.c

300

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

305

bus_addr = (iommu->tbl.table_map_base +

arch/sparc/kernel/iommu.c

306

((base - iommu->page_table) << IO_PAGE_SHIFT));

arch/sparc/kernel/iommu.c

323

iommu_free_ctx(iommu, ctx);

arch/sparc/kernel/iommu.c

330

static void strbuf_flush(struct strbuf *strbuf, struct iommu *iommu,

arch/sparc/kernel/iommu.c

337

iommu->iommu_ctxflush) {

arch/sparc/kernel/iommu.c

380

(void) iommu_read(iommu->write_complete_reg);

arch/sparc/kernel/iommu.c

400

struct iommu *iommu;

arch/sparc/kernel/iommu.c

411

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

416

base = iommu->page_table +

arch/sparc/kernel/iommu.c

417

((bus_addr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT);

arch/sparc/kernel/iommu.c

420

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

424

if (iommu->iommu_ctxflush)

arch/sparc/kernel/iommu.c

429

strbuf_flush(strbuf, iommu, bus_addr, ctx,

arch/sparc/kernel/iommu.c

434

iopte_make_dummy(iommu, base + i);

arch/sparc/kernel/iommu.c

436

iommu_free_ctx(iommu, ctx);

arch/sparc/kernel/iommu.c

437

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

439

iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE);

arch/sparc/kernel/iommu.c

453

struct iommu *iommu;

arch/sparc/kernel/iommu.c

458

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

460

if (nelems == 0 || !iommu)

arch/sparc/kernel/iommu.c

463

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

466

if (iommu->iommu_ctxflush)

arch/sparc/kernel/iommu.c

467

ctx = iommu_alloc_ctx(iommu);

arch/sparc/kernel/iommu.c

486

base_shift = iommu->tbl.table_map_base >> IO_PAGE_SHIFT;

arch/sparc/kernel/iommu.c

500

entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages,

arch/sparc/kernel/iommu.c

507

" npages %lx\n", iommu, paddr, npages);

arch/sparc/kernel/iommu.c

511

base = iommu->page_table + entry;

arch/sparc/kernel/iommu.c

514

dma_addr = iommu->tbl.table_map_base +

arch/sparc/kernel/iommu.c

52

struct iommu *iommu = container_of(iommu_map_table, struct iommu, tbl);

arch/sparc/kernel/iommu.c

53

if (iommu->iommu_flushinv) {

arch/sparc/kernel/iommu.c

54

iommu_write(iommu->iommu_flushinv, ~(u64)0);

arch/sparc/kernel/iommu.c

555

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

574

entry = (vaddr - iommu->tbl.table_map_base)

arch/sparc/kernel/iommu.c

576

base = iommu->page_table + entry;

arch/sparc/kernel/iommu.c

579

iopte_make_dummy(iommu, base + j);

arch/sparc/kernel/iommu.c

581

iommu_tbl_range_free(&iommu->tbl, vaddr, npages,

arch/sparc/kernel/iommu.c

589

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

59

tag = iommu->iommu_tags;

arch/sparc/kernel/iommu.c

597

static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg)

arch/sparc/kernel/iommu.c

601

if (iommu->iommu_ctxflush) {

arch/sparc/kernel/iommu.c

604

struct iommu_map_table *tbl = &iommu->tbl;

arch/sparc/kernel/iommu.c

607

base = iommu->page_table +

arch/sparc/kernel/iommu.c

622

struct iommu *iommu;

arch/sparc/kernel/iommu.c

626

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

629

ctx = fetch_sg_ctx(iommu, sglist);

arch/sparc/kernel/iommu.c

631

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

645

entry = ((dma_handle - iommu->tbl.table_map_base)

arch/sparc/kernel/iommu.c

647

base = iommu->page_table + entry;

arch/sparc/kernel/iommu.c

651

strbuf_flush(strbuf, iommu, dma_handle, ctx,

arch/sparc/kernel/iommu.c

655

iopte_make_dummy(iommu, base + i);

arch/sparc/kernel/iommu.c

657

iommu_tbl_range_free(&iommu->tbl, dma_handle, npages,

arch/sparc/kernel/iommu.c

66

(void) iommu_read(iommu->write_complete_reg);

arch/sparc/kernel/iommu.c

662

iommu_free_ctx(iommu, ctx);

arch/sparc/kernel/iommu.c

664

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

671

struct iommu *iommu;

arch/sparc/kernel/iommu.c

675

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

681

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

689

if (iommu->iommu_ctxflush &&

arch/sparc/kernel/iommu.c

692

struct iommu_map_table *tbl = &iommu->tbl;

arch/sparc/kernel/iommu.c

694

iopte = iommu->page_table +

arch/sparc/kernel/iommu.c

700

strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);

arch/sparc/kernel/iommu.c

702

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

709

struct iommu *iommu;

arch/sparc/kernel/iommu.c

715

iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

721

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

725

if (iommu->iommu_ctxflush &&

arch/sparc/kernel/iommu.c

728

struct iommu_map_table *tbl = &iommu->tbl;

arch/sparc/kernel/iommu.c

730

iopte = iommu->page_table + ((sglist[0].dma_address -

arch/sparc/kernel/iommu.c

746

strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);

arch/sparc/kernel/iommu.c

748

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/iommu.c

753

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/iommu.c

758

if (device_mask < iommu->dma_addr_mask)

arch/sparc/kernel/iommu.c

80

#define IOPTE_IS_DUMMY(iommu, iopte) \

arch/sparc/kernel/iommu.c

81

((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)

arch/sparc/kernel/iommu.c

83

static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte)

arch/sparc/kernel/iommu.c

88

val |= iommu->dummy_page_pa;

arch/sparc/kernel/iommu.c

93

int iommu_table_init(struct iommu *iommu, int tsbsize,

arch/sparc/kernel/ldc.c

1022

static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,

arch/sparc/kernel/ldc.c

1029

base = iommu->page_table + entry;

arch/sparc/kernel/ldc.c

1044

struct ldc_iommu *ldc_iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

1045

struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;

arch/sparc/kernel/ldc.c

1056

iommu->map = kzalloc(sz, GFP_KERNEL);

arch/sparc/kernel/ldc.c

1057

if (!iommu->map) {

arch/sparc/kernel/ldc.c

1061

iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,

arch/sparc/kernel/ldc.c

1094

kfree(iommu->map);

arch/sparc/kernel/ldc.c

1095

iommu->map = NULL;

arch/sparc/kernel/ldc.c

1102

struct ldc_iommu *ldc_iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

1103

struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;

arch/sparc/kernel/ldc.c

1108

num_tsb_entries = iommu->poolsize * iommu->nr_pools;

arch/sparc/kernel/ldc.c

1115

kfree(iommu->map);

arch/sparc/kernel/ldc.c

1116

iommu->map = NULL;

arch/sparc/kernel/ldc.c

146

struct ldc_iommu iommu;

arch/sparc/kernel/ldc.c

2017

static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,

arch/sparc/kernel/ldc.c

2022

entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,

arch/sparc/kernel/ldc.c

2027

return iommu->page_table + entry;

arch/sparc/kernel/ldc.c

2155

struct ldc_iommu *iommu;

arch/sparc/kernel/ldc.c

2170

iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

2172

base = alloc_npages(iommu, npages);

arch/sparc/kernel/ldc.c

2177

state.page_table = iommu->page_table;

arch/sparc/kernel/ldc.c

2181

state.pte_idx = (base - iommu->page_table);

arch/sparc/kernel/ldc.c

2201

struct ldc_iommu *iommu;

arch/sparc/kernel/ldc.c

2212

iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

2214

base = alloc_npages(iommu, npages);

arch/sparc/kernel/ldc.c

2219

state.page_table = iommu->page_table;

arch/sparc/kernel/ldc.c

2223

state.pte_idx = (base - iommu->page_table);

arch/sparc/kernel/ldc.c

2233

static void free_npages(unsigned long id, struct ldc_iommu *iommu,

arch/sparc/kernel/ldc.c

2240

entry = ldc_cookie_to_index(cookie, iommu);

arch/sparc/kernel/ldc.c

2241

ldc_demap(iommu, id, cookie, entry, npages);

arch/sparc/kernel/ldc.c

2242

iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);

arch/sparc/kernel/ldc.c

2248

struct ldc_iommu *iommu = &lp->iommu;

arch/sparc/kernel/ldc.c

2252

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/ldc.c

2257

free_npages(lp->id, iommu, addr, size);

arch/sparc/kernel/ldc.c

2259

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/of_device_common.c

65

op->dev.archdata.iommu = bus_sd->iommu;

arch/sparc/kernel/pci.c

275

static void pci_init_dev_archdata(struct dev_archdata *sd, void *iommu,

arch/sparc/kernel/pci.c

280

sd->iommu = iommu;

arch/sparc/kernel/pci.c

302

pci_init_dev_archdata(sd, pbm->iommu, &pbm->stc, pbm, op,

arch/sparc/kernel/pci.c

305

sd->iommu = pbm->iommu;

arch/sparc/kernel/pci.c

824

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/pci.c

841

if (iommu->dma_addr_mask & 0x80000000)

arch/sparc/kernel/pci.c

886

pci_init_dev_archdata(&dev->dev.archdata, psd->iommu,

arch/sparc/kernel/pci_fire.c

33

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_fire.c

45

iommu->iommu_control = pbm->pbm_regs + FIRE_IOMMU_CONTROL;

arch/sparc/kernel/pci_fire.c

46

iommu->iommu_tsbbase = pbm->pbm_regs + FIRE_IOMMU_TSBBASE;

arch/sparc/kernel/pci_fire.c

464

struct iommu *iommu;

arch/sparc/kernel/pci_fire.c

47

iommu->iommu_flush = pbm->pbm_regs + FIRE_IOMMU_FLUSH;

arch/sparc/kernel/pci_fire.c

477

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_fire.c

478

if (!iommu) {

arch/sparc/kernel/pci_fire.c

48

iommu->iommu_flushinv = pbm->pbm_regs + FIRE_IOMMU_FLUSHINV;

arch/sparc/kernel/pci_fire.c

483

pbm->iommu = iommu;

arch/sparc/kernel/pci_fire.c

494

kfree(pbm->iommu);

arch/sparc/kernel/pci_fire.c

53

iommu->write_complete_reg = pbm->controller_regs + 0x410000UL;

arch/sparc/kernel/pci_fire.c

58

upa_writeq(~(u64)0, iommu->iommu_flushinv);

arch/sparc/kernel/pci_fire.c

60

err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask,

arch/sparc/kernel/pci_fire.c

65

upa_writeq(__pa(iommu->page_table) | 0x7UL, iommu->iommu_tsbbase);

arch/sparc/kernel/pci_fire.c

67

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_fire.c

72

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_impl.h

150

struct iommu *iommu;

arch/sparc/kernel/pci_psycho.c

515

struct iommu *iommu;

arch/sparc/kernel/pci_psycho.c

530

iommu = pbm->sibling->iommu;

arch/sparc/kernel/pci_psycho.c

532

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_psycho.c

533

if (!iommu) {

arch/sparc/kernel/pci_psycho.c

539

pbm->iommu = iommu;

arch/sparc/kernel/pci_psycho.c

590

kfree(pbm->iommu);

arch/sparc/kernel/pci_sabre.c

466

struct iommu *iommu;

arch/sparc/kernel/pci_sabre.c

491

iommu = kzalloc_obj(*iommu);

arch/sparc/kernel/pci_sabre.c

492

if (!iommu) {

arch/sparc/kernel/pci_sabre.c

497

pbm->iommu = iommu;

arch/sparc/kernel/pci_sabre.c

580

kfree(pbm->iommu);

arch/sparc/kernel/pci_schizo.c

1140

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_schizo.c

1173

iommu->iommu_control = pbm->pbm_regs + SCHIZO_IOMMU_CONTROL;

arch/sparc/kernel/pci_schizo.c

1174

iommu->iommu_tsbbase = pbm->pbm_regs + SCHIZO_IOMMU_TSBBASE;

arch/sparc/kernel/pci_schizo.c

1175

iommu->iommu_flush = pbm->pbm_regs + SCHIZO_IOMMU_FLUSH;

arch/sparc/kernel/pci_schizo.c

1176

iommu->iommu_tags = iommu->iommu_flush + (0xa580UL - 0x0210UL);

arch/sparc/kernel/pci_schizo.c

1177

iommu->iommu_ctxflush = pbm->pbm_regs + SCHIZO_IOMMU_CTXFLUSH;

arch/sparc/kernel/pci_schizo.c

1182

iommu->write_complete_reg = pbm->controller_regs + 0x10000UL;

arch/sparc/kernel/pci_schizo.c

1187

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1189

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1201

err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask,

arch/sparc/kernel/pci_schizo.c

1208

upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);

arch/sparc/kernel/pci_schizo.c

1210

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1222

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

1422

struct iommu *iommu;

arch/sparc/kernel/pci_schizo.c

1437

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_schizo.c

1438

if (!iommu) {

arch/sparc/kernel/pci_schizo.c

1443

pbm->iommu = iommu;

arch/sparc/kernel/pci_schizo.c

1456

kfree(pbm->iommu);

arch/sparc/kernel/pci_schizo.c

242

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_schizo.c

249

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/pci_schizo.c

250

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

257

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

288

iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

304

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/pci_schizo.c

346

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/pci_sun4v.c

102

if (!iommu_use_atu(pbm->iommu, mask)) {

arch/sparc/kernel/pci_sun4v.c

119

iotsb_num = pbm->iommu->atu->iotsb->iotsb_num;

arch/sparc/kernel/pci_sun4v.c

1215

if (pbm->iommu->atu) {

arch/sparc/kernel/pci_sun4v.c

1218

kfree(pbm->iommu->atu);

arch/sparc/kernel/pci_sun4v.c

1219

pbm->iommu->atu = NULL;

arch/sparc/kernel/pci_sun4v.c

1236

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

1303

iommu = kzalloc_obj(struct iommu);

arch/sparc/kernel/pci_sun4v.c

1304

if (!iommu) {

arch/sparc/kernel/pci_sun4v.c

1309

pbm->iommu = iommu;

arch/sparc/kernel/pci_sun4v.c

1310

iommu->atu = NULL;

arch/sparc/kernel/pci_sun4v.c

1316

iommu->atu = atu;

arch/sparc/kernel/pci_sun4v.c

1328

kfree(iommu->atu);

arch/sparc/kernel/pci_sun4v.c

1329

kfree(pbm->iommu);

arch/sparc/kernel/pci_sun4v.c

188

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

213

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

215

if (!iommu_use_atu(iommu, mask))

arch/sparc/kernel/pci_sun4v.c

216

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

218

tbl = &iommu->atu->tbl;

arch/sparc/kernel/pci_sun4v.c

327

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

335

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

337

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

340

if (!iommu_use_atu(iommu, dvma)) {

arch/sparc/kernel/pci_sun4v.c

341

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

359

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

378

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

379

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

389

if (!iommu_use_atu(iommu, mask))

arch/sparc/kernel/pci_sun4v.c

390

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

443

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

457

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

459

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

468

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

488

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

497

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

498

if (nelems == 0 || !iommu)

arch/sparc/kernel/pci_sun4v.c

500

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

525

if (!iommu_use_atu(iommu, mask))

arch/sparc/kernel/pci_sun4v.c

526

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

639

struct iommu *iommu;

arch/sparc/kernel/pci_sun4v.c

647

iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

649

atu = iommu->atu;

arch/sparc/kernel/pci_sun4v.c

668

tbl = &iommu->tbl;

arch/sparc/kernel/pci_sun4v.c

686

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/pci_sun4v.c

690

if (device_mask < iommu->dma_addr_mask)

arch/sparc/kernel/pci_sun4v.c

719

struct iommu_map_table *iommu)

arch/sparc/kernel/pci_sun4v.c

726

for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) {

arch/sparc/kernel/pci_sun4v.c

727

pool = &(iommu->pools[pool_nr]);

arch/sparc/kernel/pci_sun4v.c

741

__set_bit(i, iommu->map);

arch/sparc/kernel/pci_sun4v.c

751

struct atu *atu = pbm->iommu->atu;

arch/sparc/kernel/pci_sun4v.c

78

static inline bool iommu_use_atu(struct iommu *iommu, u64 mask)

arch/sparc/kernel/pci_sun4v.c

80

return iommu->atu && mask > DMA_BIT_MASK(32);

arch/sparc/kernel/pci_sun4v.c

812

struct atu *atu = pbm->iommu->atu;

arch/sparc/kernel/pci_sun4v.c

878

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/pci_sun4v.c

899

spin_lock_init(&iommu->lock);

arch/sparc/kernel/pci_sun4v.c

900

iommu->ctx_lowest_free = 1;

arch/sparc/kernel/pci_sun4v.c

901

iommu->tbl.table_map_base = dma_offset;

arch/sparc/kernel/pci_sun4v.c

902

iommu->dma_addr_mask = dma_mask;

arch/sparc/kernel/pci_sun4v.c

907

iommu->tbl.map = kzalloc(sz, GFP_KERNEL);

arch/sparc/kernel/pci_sun4v.c

908

if (!iommu->tbl.map) {

arch/sparc/kernel/pci_sun4v.c

912

iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,

arch/sparc/kernel/pci_sun4v.c

916

sz = probe_existing_entries(pbm, &iommu->tbl);

arch/sparc/kernel/psycho_common.c

209

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/psycho_common.c

212

spin_lock_irqsave(&iommu->lock, flags);

arch/sparc/kernel/psycho_common.c

213

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

218

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

248

spin_unlock_irqrestore(&iommu->lock, flags);

arch/sparc/kernel/psycho_common.c

405

struct iommu *iommu = pbm->iommu;

arch/sparc/kernel/psycho_common.c

409

iommu->iommu_control = pbm->controller_regs + PSYCHO_IOMMU_CONTROL;

arch/sparc/kernel/psycho_common.c

410

iommu->iommu_tsbbase = pbm->controller_regs + PSYCHO_IOMMU_TSBBASE;

arch/sparc/kernel/psycho_common.c

411

iommu->iommu_flush = pbm->controller_regs + PSYCHO_IOMMU_FLUSH;

arch/sparc/kernel/psycho_common.c

412

iommu->iommu_tags = pbm->controller_regs + PSYCHO_IOMMU_TAG;

arch/sparc/kernel/psycho_common.c

413

iommu->write_complete_reg = (pbm->controller_regs +

arch/sparc/kernel/psycho_common.c

416

iommu->iommu_ctxflush = 0;

arch/sparc/kernel/psycho_common.c

418

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

420

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

425

err = iommu_table_init(iommu, tsbsize * 1024 * 8,

arch/sparc/kernel/psycho_common.c

430

upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);

arch/sparc/kernel/psycho_common.c

432

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/psycho_common.c

447

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/sbus.c

213

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

214

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

275

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

276

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

349

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

350

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

428

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

433

reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

497

struct iommu *iommu = op->dev.archdata.iommu;

arch/sparc/kernel/sbus.c

498

unsigned long reg_base = iommu->write_complete_reg - 0x2000UL;

arch/sparc/kernel/sbus.c

535

control = upa_readq(iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

537

upa_writeq(control, iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

545

struct iommu *iommu;

arch/sparc/kernel/sbus.c

559

iommu = kzalloc_obj(*iommu, GFP_ATOMIC);

arch/sparc/kernel/sbus.c

561

if (!iommu || !strbuf)

arch/sparc/kernel/sbus.c

564

op->dev.archdata.iommu = iommu;

arch/sparc/kernel/sbus.c

569

iommu->iommu_control = reg_base + IOMMU_CONTROL;

arch/sparc/kernel/sbus.c

570

iommu->iommu_tsbbase = reg_base + IOMMU_TSBBASE;

arch/sparc/kernel/sbus.c

571

iommu->iommu_flush = reg_base + IOMMU_FLUSH;

arch/sparc/kernel/sbus.c

572

iommu->iommu_tags = iommu->iommu_control +

arch/sparc/kernel/sbus.c

592

iommu->write_complete_reg = regs + 0x2000UL;

arch/sparc/kernel/sbus.c

599

if (iommu_table_init(iommu, IO_TSB_SIZE, MAP_BASE, 0xffffffff, -1))

arch/sparc/kernel/sbus.c

602

control = upa_readq(iommu->iommu_control);

arch/sparc/kernel/sbus.c

607

upa_writeq(control, iommu->iommu_control);

arch/sparc/kernel/sbus.c

615

dram = iommu->iommu_control + (IOMMU_DRAMDIAG - IOMMU_CONTROL);

arch/sparc/kernel/sbus.c

616

tag = iommu->iommu_control + (IOMMU_TAGDIAG - IOMMU_CONTROL);

arch/sparc/kernel/sbus.c

623

upa_readq(iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

626

upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);

arch/sparc/kernel/sbus.c

63

struct iommu *iommu = dev->archdata.iommu;

arch/sparc/kernel/sbus.c

648

control = upa_readq(iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

650

upa_writeq(control, iommu->write_complete_reg);

arch/sparc/kernel/sbus.c

660

kfree(iommu);

arch/sparc/kernel/sbus.c

78

cfg_reg = iommu->write_complete_reg;

arch/sparc/mm/io-unit.c

150

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

167

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

186

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

201

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

221

struct iounit_struct *iounit = dev->archdata.iommu;

arch/sparc/mm/io-unit.c

66

op->dev.archdata.iommu = iounit;

arch/sparc/mm/iommu.c

102

iommu->page_table = (iopte_t *)tmp;

arch/sparc/mm/iommu.c

105

memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));

arch/sparc/mm/iommu.c

109

base = __pa((unsigned long)iommu->page_table) >> 4;

arch/sparc/mm/iommu.c

110

sbus_writel(base, &iommu->regs->base);

arch/sparc/mm/iommu.c

111

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

119

bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);

arch/sparc/mm/iommu.c

124

iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;

arch/sparc/mm/iommu.c

126

iommu->usemap.num_colors = 1;

arch/sparc/mm/iommu.c

129

impl, vers, iommu->page_table,

arch/sparc/mm/iommu.c

132

op->dev.archdata.iommu = iommu;

arch/sparc/mm/iommu.c

187

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

216

ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);

arch/sparc/mm/iommu.c

219

busa0 = iommu->start + (ioptex << PAGE_SHIFT);

arch/sparc/mm/iommu.c

220

iopte0 = &iommu->page_table[ioptex];

arch/sparc/mm/iommu.c

226

iommu_invalidate_page(iommu->regs, busa);

arch/sparc/mm/iommu.c

285

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

289

unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;

arch/sparc/mm/iommu.c

292

BUG_ON(busa < iommu->start);

arch/sparc/mm/iommu.c

294

iopte_val(iommu->page_table[ioptex + i]) = 0;

arch/sparc/mm/iommu.c

295

iommu_invalidate_page(iommu->regs, busa);

arch/sparc/mm/iommu.c

298

bit_map_clear(&iommu->usemap, ioptex, npages);

arch/sparc/mm/iommu.c

318

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

320

iopte_t *iopte = iommu->page_table;

arch/sparc/mm/iommu.c

342

ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,

arch/sparc/mm/iommu.c

387

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

389

*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);

arch/sparc/mm/iommu.c

400

struct iommu_struct *iommu = dev->archdata.iommu;

arch/sparc/mm/iommu.c

401

iopte_t *iopte = iommu->page_table;

arch/sparc/mm/iommu.c

403

int ioptex = (busa - iommu->start) >> PAGE_SHIFT;

arch/sparc/mm/iommu.c

419

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

420

bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);

arch/sparc/mm/iommu.c

60

struct iommu_struct *iommu;

arch/sparc/mm/iommu.c

67

iommu = kmalloc_obj(struct iommu_struct);

arch/sparc/mm/iommu.c

68

if (!iommu) {

arch/sparc/mm/iommu.c

73

iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,

arch/sparc/mm/iommu.c

75

if (!iommu->regs) {

arch/sparc/mm/iommu.c

80

control = sbus_readl(&iommu->regs->control);

arch/sparc/mm/iommu.c

85

sbus_writel(control, &iommu->regs->control);

arch/sparc/mm/iommu.c

87

iommu_invalidate(iommu->regs);

arch/sparc/mm/iommu.c

88

iommu->start = IOMMU_START;

arch/sparc/mm/iommu.c

89

iommu->end = 0xffffffff;

arch/x86/events/amd/iommu.c

236

return (container_of(ev->pmu, struct perf_amd_iommu, pmu))->iommu;

arch/x86/events/amd/iommu.c

241

struct amd_iommu *iommu = perf_event_2_iommu(ev);

arch/x86/events/amd/iommu.c

248

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_COUNTER_SRC_REG, &reg);

arch/x86/events/amd/iommu.c

254

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DEVID_MATCH_REG, &reg);

arch/x86/events/amd/iommu.c

260

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_PASID_MATCH_REG, &reg);

arch/x86/events/amd/iommu.c

266

amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DOMID_MATCH_REG, &reg);

arch/x86/events/amd/iommu.c

271

struct amd_iommu *iommu = perf_event_2_iommu(event);

arch/x86/events/amd/iommu.c

275

amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,

arch/x86/events/amd/iommu.c

298

struct amd_iommu *iommu = perf_event_2_iommu(event);

arch/x86/events/amd/iommu.c

304

amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,

arch/x86/events/amd/iommu.c

315

struct amd_iommu *iommu = perf_event_2_iommu(event);

arch/x86/events/amd/iommu.c

317

if (amd_iommu_pc_get_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,

arch/x86/events/amd/iommu.c

40

struct amd_iommu *iommu;

arch/x86/events/amd/iommu.c

432

perf_iommu->iommu = get_amd_iommu(idx);

arch/x86/events/amd/iommu.c

436

if (!perf_iommu->iommu ||

arch/x86/include/asm/pci.h

21

void *iommu; /* IOMMU private data */

arch/x86/include/asm/x86_init.h

184

struct x86_init_iommu iommu;

arch/x86/kernel/aperture_64.c

431

x86_init.iommu.iommu_init = gart_iommu_init;

arch/x86/kernel/pci-dma.c

174

x86_init.iommu.iommu_init();

arch/x86/kernel/x86_init.c

107

.iommu = {

drivers/acpi/arm64/iort.c

1101

struct acpi_iort_node *iommu,

drivers/acpi/arm64/iort.c

1135

if (parent != iommu)

drivers/acpi/arm64/iort.c

1159

static void iort_find_rmrs(struct acpi_iort_node *iommu, struct device *dev,

drivers/acpi/arm64/iort.c

1183

iort_node_get_rmr_info(iort_node, iommu, dev, head);

drivers/acpi/arm64/iort.c

1199

struct acpi_iort_node *iommu;

drivers/acpi/arm64/iort.c

1201

iommu = iort_get_iort_node(iommu_fwnode);

drivers/acpi/arm64/iort.c

1202

if (!iommu)

drivers/acpi/arm64/iort.c

1205

iort_find_rmrs(iommu, dev, head);

drivers/acpi/arm64/iort.c

1210

struct acpi_iort_node *iommu;

drivers/acpi/arm64/iort.c

1213

iommu = iort_get_iort_node(fwspec->iommu_fwnode);

drivers/acpi/arm64/iort.c

1215

if (iommu && (iommu->type == ACPI_IORT_NODE_SMMU_V3)) {

drivers/acpi/arm64/iort.c

1218

smmu = (struct acpi_iort_smmu_v3 *)iommu->node_data;

drivers/acpi/arm64/iort.c

1220

return iommu;

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

250

mdev->iommu = iommu_get_domain_for_dev(mdev->dev);

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

251

if (!mdev->iommu)

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

324

if (mdev->iommu && mdev->funcs->connect_iommu)

drivers/gpu/drm/arm/display/komeda/komeda_dev.c

333

if (mdev->iommu && mdev->funcs->disconnect_iommu)

drivers/gpu/drm/arm/display/komeda/komeda_dev.h

198

struct iommu_domain *iommu;

drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c

202

kfb->is_va = mdev->iommu ? true : false;

drivers/gpu/drm/i915/i915_gpu_error.c

2063

error->iommu = i915_vtd_active(i915);

drivers/gpu/drm/i915/i915_gpu_error.c

929

err_printf(m, "IOMMU enabled?: %d\n", error->iommu);

drivers/gpu/drm/i915/i915_gpu_error.h

204

int iommu;

drivers/gpu/drm/i915/selftests/mock_gem_device.c

162

pdev->dev.iommu = &fake_iommu;

drivers/gpu/drm/msm/msm_iommu.c

138

struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);

drivers/gpu/drm/msm/msm_iommu.c

139

phys_addr_t phys = page_to_phys(iommu->prr_page);

drivers/gpu/drm/msm/msm_iommu.c

224

struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);

drivers/gpu/drm/msm/msm_iommu.c

232

mutex_lock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

233

if (--iommu->pagetables == 0) {

drivers/gpu/drm/msm/msm_iommu.c

238

__free_page(iommu->prr_page);

drivers/gpu/drm/msm/msm_iommu.c

239

iommu->prr_page = NULL;

drivers/gpu/drm/msm/msm_iommu.c

242

mutex_unlock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

269

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

271

return &iommu->domain->geometry;

drivers/gpu/drm/msm/msm_iommu.c

510

struct msm_iommu *iommu = to_msm_iommu(parent);

drivers/gpu/drm/msm/msm_iommu.c

583

mutex_lock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

584

if (iommu->pagetables++ == 0) {

drivers/gpu/drm/msm/msm_iommu.c

587

iommu->pagetables--;

drivers/gpu/drm/msm/msm_iommu.c

588

mutex_unlock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

594

BUG_ON(iommu->prr_page);

drivers/gpu/drm/msm/msm_iommu.c

607

iommu->prr_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

drivers/gpu/drm/msm/msm_iommu.c

609

page_to_phys(iommu->prr_page));

drivers/gpu/drm/msm/msm_iommu.c

613

mutex_unlock(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

635

struct msm_iommu *iommu = arg;

drivers/gpu/drm/msm/msm_iommu.c

636

struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(iommu->base.dev);

drivers/gpu/drm/msm/msm_iommu.c

644

if (iommu->base.handler)

drivers/gpu/drm/msm/msm_iommu.c

645

return iommu->base.handler(iommu->base.arg, iova, flags, ptr);

drivers/gpu/drm/msm/msm_iommu.c

655

struct msm_iommu *iommu = arg;

drivers/gpu/drm/msm/msm_iommu.c

657

if (iommu->base.handler)

drivers/gpu/drm/msm/msm_iommu.c

658

return iommu->base.handler(iommu->base.arg, iova, flags, NULL);

drivers/gpu/drm/msm/msm_iommu.c

673

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

675

iommu_detach_device(iommu->domain, mmu->dev);

drivers/gpu/drm/msm/msm_iommu.c

682

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

691

ret = iommu_map_sgtable(iommu->domain, iova, sgt, prot);

drivers/gpu/drm/msm/msm_iommu.c

700

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

705

iommu_unmap(iommu->domain, iova, len);

drivers/gpu/drm/msm/msm_iommu.c

712

struct msm_iommu *iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

713

iommu_domain_free(iommu->domain);

drivers/gpu/drm/msm/msm_iommu.c

714

kmem_cache_destroy(iommu->pt_cache);

drivers/gpu/drm/msm/msm_iommu.c

715

kfree(iommu);

drivers/gpu/drm/msm/msm_iommu.c

729

struct msm_iommu *iommu;

drivers/gpu/drm/msm/msm_iommu.c

741

iommu = kzalloc_obj(*iommu);

drivers/gpu/drm/msm/msm_iommu.c

742

if (!iommu) {

drivers/gpu/drm/msm/msm_iommu.c

747

iommu->domain = domain;

drivers/gpu/drm/msm/msm_iommu.c

748

msm_mmu_init(&iommu->base, dev, &funcs, MSM_MMU_IOMMU);

drivers/gpu/drm/msm/msm_iommu.c

750

mutex_init(&iommu->init_lock);

drivers/gpu/drm/msm/msm_iommu.c

752

ret = iommu_attach_device(iommu->domain, dev);

drivers/gpu/drm/msm/msm_iommu.c

755

kfree(iommu);

drivers/gpu/drm/msm/msm_iommu.c

759

return &iommu->base;

drivers/gpu/drm/msm/msm_iommu.c

764

struct msm_iommu *iommu;

drivers/gpu/drm/msm/msm_iommu.c

771

iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

772

iommu_set_fault_handler(iommu->domain, msm_disp_fault_handler, iommu);

drivers/gpu/drm/msm/msm_iommu.c

780

struct msm_iommu *iommu;

drivers/gpu/drm/msm/msm_iommu.c

787

iommu = to_msm_iommu(mmu);

drivers/gpu/drm/msm/msm_iommu.c

793

iommu->pt_cache =

drivers/gpu/drm/msm/msm_iommu.c

796

iommu_set_fault_handler(iommu->domain, msm_gpu_fault_handler, iommu);

drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h

32

} iommu;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

121

mutex_init(&tdev->iommu.mutex);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

124

tdev->iommu.domain = iommu_paging_domain_alloc(dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

125

if (IS_ERR(tdev->iommu.domain))

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

133

pgsize_bitmap = tdev->iommu.domain->pgsize_bitmap;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

135

tdev->iommu.pgshift = PAGE_SHIFT;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

137

tdev->iommu.pgshift = fls(pgsize_bitmap & ~PAGE_MASK);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

138

if (tdev->iommu.pgshift == 0) {

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

142

tdev->iommu.pgshift -= 1;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

145

ret = iommu_attach_device(tdev->iommu.domain, dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

149

ret = nvkm_mm_init(&tdev->iommu.mm, 0, 0,

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

151

tdev->iommu.pgshift, 1);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

159

iommu_detach_device(tdev->iommu.domain, dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

162

iommu_domain_free(tdev->iommu.domain);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

165

tdev->iommu.domain = NULL;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

166

tdev->iommu.pgshift = 0;

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

175

if (tdev->iommu.domain) {

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

176

nvkm_mm_fini(&tdev->iommu.mm);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

177

iommu_detach_device(tdev->iommu.domain, tdev->device.dev);

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c

178

iommu_domain_free(tdev->iommu.domain);

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

591

if (tdev->iommu.domain) {

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

592

imem->mm_mutex = &tdev->iommu.mutex;

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

593

imem->mm = &tdev->iommu.mm;

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

594

imem->domain = tdev->iommu.domain;

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

595

imem->iommu_pgshift = tdev->iommu.pgshift;

drivers/gpu/host1x/context.c

136

if (cd->dev.iommu->iommu_dev != dev->iommu->iommu_dev)

drivers/iommu/amd/amd_iommu.h

101

static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu)

drivers/iommu/amd/amd_iommu.h

161

return iommu_get_iommu_dev(dev, struct amd_iommu, iommu);

drivers/iommu/amd/amd_iommu.h

167

return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu);

drivers/iommu/amd/amd_iommu.h

175

bool translation_pre_enabled(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

192

struct dev_table_entry *get_dev_table(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

193

struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/amd_iommu.h

199

void amd_iommu_update_dte(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

207

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/amd_iommu.h

21

void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,

drivers/iommu/amd/amd_iommu.h

217

initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);

drivers/iommu/amd/amd_iommu.h

24

void amd_iommu_restart_event_logging(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

25

void amd_iommu_restart_ga_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

26

void amd_iommu_restart_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

27

void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/amd_iommu.h

28

void iommu_feature_enable(struct amd_iommu *iommu, u8 bit);

drivers/iommu/amd/amd_iommu.h

29

void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

67

int amd_iommu_iopf_init(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

68

void amd_iommu_iopf_uninit(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

71

int amd_iommu_iopf_add_device(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

73

void amd_iommu_iopf_remove_device(struct amd_iommu *iommu,

drivers/iommu/amd/amd_iommu.h

82

int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

83

void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

84

void amd_iommu_enable_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

85

void amd_iommu_poll_ppr_log(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

92

void amd_iommu_flush_all_caches(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu.h

99

int amd_iommu_create_irq_domain(struct amd_iommu *iommu);

drivers/iommu/amd/amd_iommu_types.h

1030

struct amd_iommu *iommu;

drivers/iommu/amd/amd_iommu_types.h

1047

void (*activate)(struct amd_iommu *iommu, void *, u16, u16);

drivers/iommu/amd/amd_iommu_types.h

1048

void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);

drivers/iommu/amd/amd_iommu_types.h

1049

void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);

drivers/iommu/amd/amd_iommu_types.h

451

#define for_each_iommu(iommu) \

drivers/iommu/amd/amd_iommu_types.h

452

list_for_each_entry((iommu), &amd_iommu_list, list)

drivers/iommu/amd/amd_iommu_types.h

453

#define for_each_iommu_safe(iommu, next) \

drivers/iommu/amd/amd_iommu_types.h

454

list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)

drivers/iommu/amd/amd_iommu_types.h

496

struct amd_iommu *iommu; /* IOMMUs attach to protection domain */

drivers/iommu/amd/amd_iommu_types.h

539

struct pt_iommu iommu;

drivers/iommu/amd/amd_iommu_types.h

559

PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);

drivers/iommu/amd/amd_iommu_types.h

560

PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);

drivers/iommu/amd/amd_iommu_types.h

561

PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv2.iommu, domain);

drivers/iommu/amd/amd_iommu_types.h

715

struct iommu_device iommu;

drivers/iommu/amd/amd_iommu_types.h

767

struct iommu_device *iommu = dev_to_iommu_device(dev);

drivers/iommu/amd/amd_iommu_types.h

769

return container_of(iommu, struct amd_iommu, iommu);

drivers/iommu/amd/debugfs.c

113

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

119

raw_spin_lock_irqsave(&iommu->lock, flag);

drivers/iommu/amd/debugfs.c

120

head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

drivers/iommu/amd/debugfs.c

121

tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/debugfs.c

125

cmd = (struct iommu_cmd *)(iommu->cmd_buf + i * sizeof(*cmd));

drivers/iommu/amd/debugfs.c

129

raw_spin_unlock_irqrestore(&iommu->lock, flag);

drivers/iommu/amd/debugfs.c

140

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

174

iommu = pci_seg->rlookup_table[devid];

drivers/iommu/amd/debugfs.c

175

if (!iommu) {

drivers/iommu/amd/debugfs.c

213

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

215

iommu = pci_seg->rlookup_table[devid];

drivers/iommu/amd/debugfs.c

216

if (!iommu)

drivers/iommu/amd/debugfs.c

219

dev_table = get_dev_table(iommu);

drivers/iommu/amd/debugfs.c

231

seq_printf(m, "iommu%d\n", iommu->index);

drivers/iommu/amd/debugfs.c

28

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

29

int ret, dbg_mmio_offset = iommu->dbg_mmio_offset = -1;

drivers/iommu/amd/debugfs.c

295

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

306

iommu = pci_seg->rlookup_table[devid];

drivers/iommu/amd/debugfs.c

307

if (!iommu)

drivers/iommu/amd/debugfs.c

310

dev_table = get_dev_table(iommu);

drivers/iommu/amd/debugfs.c

366

struct amd_iommu *iommu;

drivers/iommu/amd/debugfs.c

371

for_each_iommu(iommu) {

drivers/iommu/amd/debugfs.c

372

iommu->dbg_mmio_offset = -1;

drivers/iommu/amd/debugfs.c

373

iommu->dbg_cap_offset = -1;

drivers/iommu/amd/debugfs.c

375

snprintf(name, MAX_NAME_LEN, "iommu%02d", iommu->index);

drivers/iommu/amd/debugfs.c

376

iommu->debugfs = debugfs_create_dir(name, amd_iommu_debugfs);

drivers/iommu/amd/debugfs.c

378

debugfs_create_file("mmio", 0644, iommu->debugfs, iommu,

drivers/iommu/amd/debugfs.c

380

debugfs_create_file("capability", 0644, iommu->debugfs, iommu,

drivers/iommu/amd/debugfs.c

382

debugfs_create_file("cmdbuf", 0444, iommu->debugfs, iommu,

drivers/iommu/amd/debugfs.c

39

iommu->mmio_phys_end - sizeof(u64))

drivers/iommu/amd/debugfs.c

42

iommu->dbg_mmio_offset = dbg_mmio_offset;

drivers/iommu/amd/debugfs.c

48

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

50

int dbg_mmio_offset = iommu->dbg_mmio_offset;

drivers/iommu/amd/debugfs.c

53

iommu->mmio_phys_end - sizeof(u64)) {

drivers/iommu/amd/debugfs.c

58

value = readq(iommu->mmio_base + dbg_mmio_offset);

drivers/iommu/amd/debugfs.c

69

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

70

int ret, dbg_cap_offset = iommu->dbg_cap_offset = -1;

drivers/iommu/amd/debugfs.c

83

iommu->dbg_cap_offset = dbg_cap_offset;

drivers/iommu/amd/debugfs.c

89

struct amd_iommu *iommu = m->private;

drivers/iommu/amd/debugfs.c

91

int err, dbg_cap_offset = iommu->dbg_cap_offset;

drivers/iommu/amd/debugfs.c

98

err = pci_read_config_dword(iommu->dev, iommu->cap_ptr + dbg_cap_offset, &value);

drivers/iommu/amd/init.c

1001

paddr = readq(iommu->mmio_base + MMIO_CMD_BUF_OFFSET) & PM_ADDR_MASK;

drivers/iommu/amd/init.c

1002

iommu->cmd_buf = iommu_memremap(paddr, CMD_BUFFER_SIZE);

drivers/iommu/amd/init.c

1004

return iommu->cmd_buf ? 0 : -ENOMEM;

drivers/iommu/amd/init.c

1007

static int __init remap_or_alloc_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1017

paddr = readq(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET) & PM_ADDR_MASK;

drivers/iommu/amd/init.c

1018

iommu->cmd_sem = iommu_memremap(paddr, PAGE_SIZE);

drivers/iommu/amd/init.c

1019

if (!iommu->cmd_sem)

drivers/iommu/amd/init.c

1021

iommu->cmd_sem_paddr = paddr;

drivers/iommu/amd/init.c

1023

return alloc_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1029

static int __init alloc_iommu_buffers(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1038

ret = remap_or_alloc_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1042

ret = remap_command_buffer(iommu);

drivers/iommu/amd/init.c

1046

ret = alloc_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1050

ret = alloc_command_buffer(iommu);

drivers/iommu/amd/init.c

1058

static void __init free_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1060

if (iommu->cmd_sem)

drivers/iommu/amd/init.c

1061

iommu_free_pages((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

1063

static void __init unmap_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1065

if (iommu->cmd_sem) {

drivers/iommu/amd/init.c

1067

memunmap((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

1069

iommu_free_pages((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

1073

static void __init unmap_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1075

memunmap((void *)iommu->cmd_buf);

drivers/iommu/amd/init.c

1078

static void __init unmap_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1080

memunmap(iommu->evt_buf);

drivers/iommu/amd/init.c

1083

static void __init free_iommu_buffers(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1086

unmap_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1087

unmap_command_buffer(iommu);

drivers/iommu/amd/init.c

1088

unmap_event_buffer(iommu);

drivers/iommu/amd/init.c

1090

free_cwwb_sem(iommu);

drivers/iommu/amd/init.c

1091

free_command_buffer(iommu);

drivers/iommu/amd/init.c

1092

free_event_buffer(iommu);

drivers/iommu/amd/init.c

1096

static void iommu_enable_xt(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1105

iommu_feature_enable(iommu, CONTROL_XT_EN);

drivers/iommu/amd/init.c

1109

static void iommu_enable_gt(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1114

iommu_feature_enable(iommu, CONTROL_GT_EN);

drivers/iommu/amd/init.c

1122

iommu_feature_enable(iommu, CONTROL_GCR3TRPMODE);

drivers/iommu/amd/init.c

1134

static bool __reuse_device_table(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1136

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/init.c

1145

lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);

drivers/iommu/amd/init.c

1146

hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);

drivers/iommu/amd/init.c

1152

iommu->index);

drivers/iommu/amd/init.c

1198

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

1211

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

1212

if (pci_seg->id != iommu->pci_seg->id)

drivers/iommu/amd/init.c

1214

if (!__reuse_device_table(iommu))

drivers/iommu/amd/init.c

1265

set_dev_entry_from_acpi_range(struct amd_iommu *iommu, u16 first, u16 last,

drivers/iommu/amd/init.c

1275

if (search_ivhd_dte_flags(iommu->pci_seg->id, first, last))

drivers/iommu/amd/init.c

1304

d->segid = iommu->pci_seg->id;

drivers/iommu/amd/init.c

1312

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/init.c

1316

amd_iommu_set_rlookup_table(iommu, i);

drivers/iommu/amd/init.c

1320

static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,

drivers/iommu/amd/init.c

1323

set_dev_entry_from_acpi_range(iommu, devid, devid, flags, ext_flags);

drivers/iommu/amd/init.c

1437

static int __init init_iommu_from_acpi(struct amd_iommu *iommu,

drivers/iommu/amd/init.c

1446

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/init.c

1460

iommu->acpi_flags = h->flags;

drivers/iommu/amd/init.c

1484

set_dev_entry_from_acpi_range(iommu, 0, pci_seg->last_bdf, e->flags, 0);

drivers/iommu/amd/init.c

1495

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

drivers/iommu/amd/init.c

1523

set_dev_entry_from_acpi(iommu, devid , e->flags, 0);

drivers/iommu/amd/init.c

1524

set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);

drivers/iommu/amd/init.c

1553

set_dev_entry_from_acpi(iommu, devid, e->flags,

drivers/iommu/amd/init.c

1580

set_dev_entry_from_acpi(iommu, devid_to, flags, ext_flags);

drivers/iommu/amd/init.c

1582

set_dev_entry_from_acpi_range(iommu, devid_start, devid, flags, ext_flags);

drivers/iommu/amd/init.c

1617

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

drivers/iommu/amd/init.c

1684

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

drivers/iommu/amd/init.c

1775

static void __init free_sysfs(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1777

if (iommu->iommu.dev) {

drivers/iommu/amd/init.c

1778

iommu_device_unregister(&iommu->iommu);

drivers/iommu/amd/init.c

1779

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/amd/init.c

1783

static void __init free_iommu_one(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1785

free_sysfs(iommu);

drivers/iommu/amd/init.c

1786

free_iommu_buffers(iommu);

drivers/iommu/amd/init.c

1787

amd_iommu_free_ppr_log(iommu);

drivers/iommu/amd/init.c

1788

free_ga_log(iommu);

drivers/iommu/amd/init.c

1789

iommu_unmap_mmio_space(iommu);

drivers/iommu/amd/init.c

1790

amd_iommu_iopf_uninit(iommu);

drivers/iommu/amd/init.c

1795

struct amd_iommu *iommu, *next;

drivers/iommu/amd/init.c

1797

for_each_iommu_safe(iommu, next) {

drivers/iommu/amd/init.c

1798

list_del(&iommu->list);

drivers/iommu/amd/init.c

1799

free_iommu_one(iommu);

drivers/iommu/amd/init.c

1800

kfree(iommu);

drivers/iommu/amd/init.c

1810

static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1819

pci_write_config_dword(iommu->dev, 0xf0, 0x90);

drivers/iommu/amd/init.c

1820

pci_read_config_dword(iommu->dev, 0xf4, &value);

drivers/iommu/amd/init.c

1826

pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));

drivers/iommu/amd/init.c

1828

pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);

drivers/iommu/amd/init.c

1829

pci_info(iommu->dev, "Applying erratum 746 workaround\n");

drivers/iommu/amd/init.c

1832

pci_write_config_dword(iommu->dev, 0xf0, 0x90);

drivers/iommu/amd/init.c

1841

static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1851

value = iommu_read_l2(iommu, 0x47);

drivers/iommu/amd/init.c

1857

iommu_write_l2(iommu, 0x47, value | BIT(0));

drivers/iommu/amd/init.c

1859

pci_info(iommu->dev, "Applying ATS write check workaround\n");

drivers/iommu/amd/init.c

1867

static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,

drivers/iommu/amd/init.c

1875

iommu->pci_seg = pci_seg;

drivers/iommu/amd/init.c

1877

raw_spin_lock_init(&iommu->lock);

drivers/iommu/amd/init.c

1878

iommu->cmd_sem_val = 0;

drivers/iommu/amd/init.c

1881

list_add_tail(&iommu->list, &amd_iommu_list);

drivers/iommu/amd/init.c

1882

iommu->index = amd_iommus_present++;

drivers/iommu/amd/init.c

1884

if (unlikely(iommu->index >= MAX_IOMMUS)) {

drivers/iommu/amd/init.c

1892

iommu->devid = h->devid;

drivers/iommu/amd/init.c

1893

iommu->cap_ptr = h->cap_ptr;

drivers/iommu/amd/init.c

1894

iommu->mmio_phys = h->mmio_phys;

drivers/iommu/amd/init.c

1902

iommu->mmio_phys_end = MMIO_REG_END_OFFSET;

drivers/iommu/amd/init.c

1904

iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;

drivers/iommu/amd/init.c

1913

iommu->mmio_phys_end = MMIO_REG_END_OFFSET;

drivers/iommu/amd/init.c

1915

iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;

drivers/iommu/amd/init.c

1930

early_iommu_features_init(iommu, h);

drivers/iommu/amd/init.c

1937

iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,

drivers/iommu/amd/init.c

1938

iommu->mmio_phys_end);

drivers/iommu/amd/init.c

1939

if (!iommu->mmio_base)

drivers/iommu/amd/init.c

1942

return init_iommu_from_acpi(iommu, h);

drivers/iommu/amd/init.c

1945

static int __init init_iommu_one_late(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

1949

ret = alloc_iommu_buffers(iommu);

drivers/iommu/amd/init.c

1953

iommu->int_enabled = false;

drivers/iommu/amd/init.c

1955

init_translation_status(iommu);

drivers/iommu/amd/init.c

1956

if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {

drivers/iommu/amd/init.c

1957

iommu_disable(iommu);

drivers/iommu/amd/init.c

1958

clear_translation_pre_enabled(iommu);

drivers/iommu/amd/init.c

1960

iommu->index);

drivers/iommu/amd/init.c

1963

amd_iommu_pre_enabled = translation_pre_enabled(iommu);

drivers/iommu/amd/init.c

1966

ret = amd_iommu_create_irq_domain(iommu);

drivers/iommu/amd/init.c

1975

iommu->pci_seg->rlookup_table[iommu->devid] = NULL;

drivers/iommu/amd/init.c

2014

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2033

iommu = kzalloc_obj(struct amd_iommu);

drivers/iommu/amd/init.c

2034

if (iommu == NULL)

drivers/iommu/amd/init.c

2037

ret = init_iommu_one(iommu, h, table);

drivers/iommu/amd/init.c

2050

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2051

ret = init_iommu_one_late(iommu);

drivers/iommu/amd/init.c

2059

static void init_iommu_perf_ctr(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2062

struct pci_dev *pdev = iommu->dev;

drivers/iommu/amd/init.c

2071

val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);

drivers/iommu/amd/init.c

2072

iommu->max_banks = (u8) ((val >> 12) & 0x3f);

drivers/iommu/amd/init.c

2073

iommu->max_counters = (u8) ((val >> 7) & 0xf);

drivers/iommu/amd/init.c

2082

struct amd_iommu *iommu = dev_to_amd_iommu(dev);

drivers/iommu/amd/init.c

2083

return sysfs_emit(buf, "%x\n", iommu->cap);

drivers/iommu/amd/init.c

2116

static void __init late_iommu_features_init(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2120

if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))

drivers/iommu/amd/init.c

2124

features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);

drivers/iommu/amd/init.c

2125

features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);

drivers/iommu/amd/init.c

2146

static int __init iommu_init_pci(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2148

int cap_ptr = iommu->cap_ptr;

drivers/iommu/amd/init.c

2151

iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,

drivers/iommu/amd/init.c

2152

PCI_BUS_NUM(iommu->devid),

drivers/iommu/amd/init.c

2153

iommu->devid & 0xff);

drivers/iommu/amd/init.c

2154

if (!iommu->dev)

drivers/iommu/amd/init.c

2158

iommu->dev->irq_managed = 1;

drivers/iommu/amd/init.c

2160

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,

drivers/iommu/amd/init.c

2161

&iommu->cap);

drivers/iommu/amd/init.c

2163

if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))

drivers/iommu/amd/init.c

2166

late_iommu_features_init(iommu);

drivers/iommu/amd/init.c

2173

iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1;

drivers/iommu/amd/init.c

2175

BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK);

drivers/iommu/amd/init.c

2184

iommu_enable_gt(iommu);

drivers/iommu/amd/init.c

2187

if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))

drivers/iommu/amd/init.c

2190

if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {

drivers/iommu/amd/init.c

2196

init_iommu_perf_ctr(iommu);

drivers/iommu/amd/init.c

2198

if (is_rd890_iommu(iommu->dev)) {

drivers/iommu/amd/init.c

2201

iommu->root_pdev =

drivers/iommu/amd/init.c

2202

pci_get_domain_bus_and_slot(iommu->pci_seg->id,

drivers/iommu/amd/init.c

2203

iommu->dev->bus->number,

drivers/iommu/amd/init.c

2211

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,

drivers/iommu/amd/init.c

2212

&iommu->stored_addr_lo);

drivers/iommu/amd/init.c

2213

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,

drivers/iommu/amd/init.c

2214

&iommu->stored_addr_hi);

drivers/iommu/amd/init.c

2217

iommu->stored_addr_lo &= ~1;

drivers/iommu/amd/init.c

2221

iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);

drivers/iommu/amd/init.c

2224

iommu->stored_l2[i] = iommu_read_l2(iommu, i);

drivers/iommu/amd/init.c

2227

amd_iommu_erratum_746_workaround(iommu);

drivers/iommu/amd/init.c

2228

amd_iommu_ats_write_check_workaround(iommu);

drivers/iommu/amd/init.c

2230

ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,

drivers/iommu/amd/init.c

2231

amd_iommu_groups, "ivhd%d", iommu->index);

drivers/iommu/amd/init.c

2240

ret = amd_iommu_iopf_init(iommu);

drivers/iommu/amd/init.c

2245

ret = iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);

drivers/iommu/amd/init.c

2252

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/amd/init.c

2255

return pci_enable_device(iommu->dev);

drivers/iommu/amd/init.c

2299

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2306

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2307

ret = iommu_init_pci(iommu);

drivers/iommu/amd/init.c

2310

iommu->index, ret);

drivers/iommu/amd/init.c

2314

iommu_set_cwwb_range(iommu);

drivers/iommu/amd/init.c

2330

for_each_iommu(iommu)

drivers/iommu/amd/init.c

2331

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

2348

static int iommu_setup_msi(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

235

bool translation_pre_enabled(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2352

r = pci_enable_msi(iommu->dev);

drivers/iommu/amd/init.c

2356

r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread,

drivers/iommu/amd/init.c

2357

IRQF_ONESHOT, "AMD-Vi", iommu);

drivers/iommu/amd/init.c

2359

pci_disable_msi(iommu->dev);

drivers/iommu/amd/init.c

237

return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);

drivers/iommu/amd/init.c

240

static void clear_translation_pre_enabled(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

242

iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/amd/init.c

2428

struct amd_iommu *iommu = irqd->chip_data;

drivers/iommu/amd/init.c

2438

writeq(xt.capxt, iommu->mmio_base + irqd->hwirq);

drivers/iommu/amd/init.c

2443

struct amd_iommu *iommu = irqd->chip_data;

drivers/iommu/amd/init.c

2445

writeq(0, iommu->mmio_base + irqd->hwirq);

drivers/iommu/amd/init.c

245

static void init_translation_status(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

249

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

2508

static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname,

drivers/iommu/amd/init.c

251

iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/amd/init.c

2514

int node = dev_to_node(&iommu->dev->dev);

drivers/iommu/amd/init.c

2522

info.data = iommu;

drivers/iommu/amd/init.c

2532

iommu);

drivers/iommu/amd/init.c

2542

static int iommu_setup_intcapxt(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2546

snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name),

drivers/iommu/amd/init.c

2547

"AMD-Vi%d-Evt", iommu->index);

drivers/iommu/amd/init.c

2548

ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name,

drivers/iommu/amd/init.c

2554

snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name),

drivers/iommu/amd/init.c

2555

"AMD-Vi%d-PPR", iommu->index);

drivers/iommu/amd/init.c

2556

ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name,

drivers/iommu/amd/init.c

2563

snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name),

drivers/iommu/amd/init.c

2564

"AMD-Vi%d-GA", iommu->index);

drivers/iommu/amd/init.c

2565

ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name,

drivers/iommu/amd/init.c

2573

static int iommu_init_irq(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2577

if (iommu->int_enabled)

drivers/iommu/amd/init.c

2581

ret = iommu_setup_intcapxt(iommu);

drivers/iommu/amd/init.c

2582

else if (iommu->dev->msi_cap)

drivers/iommu/amd/init.c

2583

ret = iommu_setup_msi(iommu);

drivers/iommu/amd/init.c

2590

iommu->int_enabled = true;

drivers/iommu/amd/init.c

2594

iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);

drivers/iommu/amd/init.c

2596

iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);

drivers/iommu/amd/init.c

270

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

272

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

273

u64 tmp = iommu->features;

drivers/iommu/amd/init.c

274

u64 tmp2 = iommu->features2;

drivers/iommu/amd/init.c

2751

static void iommu_init_flags(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2753

iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?

drivers/iommu/amd/init.c

2754

iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :

drivers/iommu/amd/init.c

2755

iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);

drivers/iommu/amd/init.c

2757

iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?

drivers/iommu/amd/init.c

2758

iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :

drivers/iommu/amd/init.c

2759

iommu_feature_disable(iommu, CONTROL_PASSPW_EN);

drivers/iommu/amd/init.c

276

if (list_is_first(&iommu->list, &amd_iommu_list)) {

drivers/iommu/amd/init.c

2761

iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?

drivers/iommu/amd/init.c

2762

iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :

drivers/iommu/amd/init.c

2763

iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);

drivers/iommu/amd/init.c

2765

iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?

drivers/iommu/amd/init.c

2766

iommu_feature_enable(iommu, CONTROL_ISOC_EN) :

drivers/iommu/amd/init.c

2767

iommu_feature_disable(iommu, CONTROL_ISOC_EN);

drivers/iommu/amd/init.c

2772

iommu_feature_enable(iommu, CONTROL_COHERENT_EN);

drivers/iommu/amd/init.c

2775

iommu_feature_set(iommu, CTRL_INV_TO_1S, CTRL_INV_TO_MASK, CONTROL_INV_TIMEOUT);

drivers/iommu/amd/init.c

2779

iommu_feature_enable(iommu, CONTROL_EPH_EN);

drivers/iommu/amd/init.c

2782

static void iommu_apply_resume_quirks(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2786

struct pci_dev *pdev = iommu->root_pdev;

drivers/iommu/amd/init.c

2789

if (!is_rd890_iommu(iommu->dev) || !pdev)

drivers/iommu/amd/init.c

2806

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

drivers/iommu/amd/init.c

2807

iommu->stored_addr_lo);

drivers/iommu/amd/init.c

2808

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,

drivers/iommu/amd/init.c

2809

iommu->stored_addr_hi);

drivers/iommu/amd/init.c

2814

iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);

drivers/iommu/amd/init.c

2818

iommu_write_l2(iommu, i, iommu->stored_l2[i]);

drivers/iommu/amd/init.c

2821

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

drivers/iommu/amd/init.c

2822

iommu->stored_addr_lo | 1);

drivers/iommu/amd/init.c

2825

static void iommu_enable_ga(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2831

iommu_feature_enable(iommu, CONTROL_GA_EN);

drivers/iommu/amd/init.c

2832

iommu->irte_ops = &irte_128_ops;

drivers/iommu/amd/init.c

2835

iommu->irte_ops = &irte_32_ops;

drivers/iommu/amd/init.c

2841

static void iommu_disable_irtcachedis(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2843

iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);

drivers/iommu/amd/init.c

2846

static void iommu_enable_irtcachedis(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2858

iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);

drivers/iommu/amd/init.c

2859

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

2862

iommu->irtcachedis_enabled = true;

drivers/iommu/amd/init.c

2864

iommu->index, iommu->devid,

drivers/iommu/amd/init.c

2865

iommu->irtcachedis_enabled ? "disabled" : "enabled");

drivers/iommu/amd/init.c

2868

static void iommu_enable_2k_int(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2873

iommu_feature_set(iommu,

drivers/iommu/amd/init.c

2879

static void early_enable_iommu(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

2881

iommu_disable(iommu);

drivers/iommu/amd/init.c

2882

iommu_init_flags(iommu);

drivers/iommu/amd/init.c

2883

iommu_set_device_table(iommu);

drivers/iommu/amd/init.c

2884

iommu_enable_command_buffer(iommu);

drivers/iommu/amd/init.c

2885

iommu_enable_gt(iommu);

drivers/iommu/amd/init.c

2886

iommu_enable_ga(iommu);

drivers/iommu/amd/init.c

2887

iommu_enable_xt(iommu);

drivers/iommu/amd/init.c

2888

iommu_enable_irtcachedis(iommu);

drivers/iommu/amd/init.c

2889

iommu_enable_2k_int(iommu);

drivers/iommu/amd/init.c

289

iommu->index, iommu->pci_seg->id,

drivers/iommu/amd/init.c

2890

iommu_enable(iommu);

drivers/iommu/amd/init.c

2891

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

290

PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),

drivers/iommu/amd/init.c

2904

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

291

PCI_FUNC(iommu->devid));

drivers/iommu/amd/init.c

2930

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2931

clear_translation_pre_enabled(iommu);

drivers/iommu/amd/init.c

2932

early_enable_iommu(iommu);

drivers/iommu/amd/init.c

2942

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2943

iommu_disable_command_buffer(iommu);

drivers/iommu/amd/init.c

2944

iommu_disable_event_buffer(iommu);

drivers/iommu/amd/init.c

2945

iommu_disable_irtcachedis(iommu);

drivers/iommu/amd/init.c

2946

iommu_enable_command_buffer(iommu);

drivers/iommu/amd/init.c

2947

iommu_enable_ga(iommu);

drivers/iommu/amd/init.c

2948

iommu_enable_xt(iommu);

drivers/iommu/amd/init.c

2949

iommu_enable_irtcachedis(iommu);

drivers/iommu/amd/init.c

2950

iommu_enable_2k_int(iommu);

drivers/iommu/amd/init.c

2951

iommu_set_device_table(iommu);

drivers/iommu/amd/init.c

2952

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

2959

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2964

for_each_iommu(iommu)

drivers/iommu/amd/init.c

2965

amd_iommu_enable_ppr_log(iommu);

drivers/iommu/amd/init.c

2972

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

2974

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

2979

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

2983

iommu_feature_disable(iommu, CONTROL_GALOG_EN);

drivers/iommu/amd/init.c

2984

iommu_feature_disable(iommu, CONTROL_GAINT_EN);

drivers/iommu/amd/init.c

2991

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

3017

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

3018

if (iommu_init_ga_log(iommu) ||

drivers/iommu/amd/init.c

3019

iommu_ga_log_enable(iommu))

drivers/iommu/amd/init.c

3022

iommu_feature_enable(iommu, CONTROL_GAM_EN);

drivers/iommu/amd/init.c

3024

iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);

drivers/iommu/amd/init.c

3034

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3036

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3037

iommu_disable(iommu);

drivers/iommu/amd/init.c

305

static void __init early_iommu_features_init(struct amd_iommu *iommu,

drivers/iommu/amd/init.c

3052

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3054

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3055

iommu_apply_resume_quirks(iommu);

drivers/iommu/amd/init.c

3058

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3059

early_enable_iommu(iommu);

drivers/iommu/amd/init.c

309

iommu->features = h->efr_reg;

drivers/iommu/amd/init.c

310

iommu->features2 = h->efr_reg2;

drivers/iommu/amd/init.c

318

static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)

drivers/iommu/amd/init.c

322

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

drivers/iommu/amd/init.c

323

pci_read_config_dword(iommu->dev, 0xfc, &val);

drivers/iommu/amd/init.c

327

static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)

drivers/iommu/amd/init.c

329

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));

drivers/iommu/amd/init.c

330

pci_write_config_dword(iommu->dev, 0xfc, val);

drivers/iommu/amd/init.c

3305

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3308

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

3309

ret = iommu_init_irq(iommu);

drivers/iommu/amd/init.c

331

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

drivers/iommu/amd/init.c

334

static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)

drivers/iommu/amd/init.c

338

pci_write_config_dword(iommu->dev, 0xf0, address);

drivers/iommu/amd/init.c

339

pci_read_config_dword(iommu->dev, 0xf4, &val);

drivers/iommu/amd/init.c

343

static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)

drivers/iommu/amd/init.c

345

pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));

drivers/iommu/amd/init.c

346

pci_write_config_dword(iommu->dev, 0xf4, val);

drivers/iommu/amd/init.c

3512

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3518

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3519

amd_iommu_flush_all_caches(iommu);

drivers/iommu/amd/init.c

358

static void iommu_set_cwwb_range(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

360

u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

3668

x86_init.iommu.iommu_init = amd_iommu_init;

drivers/iommu/amd/init.c

370

memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,

drivers/iommu/amd/init.c

377

memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,

drivers/iommu/amd/init.c

382

static void iommu_set_device_table(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

385

u32 dev_table_size = iommu->pci_seg->dev_table_size;

drivers/iommu/amd/init.c

386

void *dev_table = (void *)get_dev_table(iommu);

drivers/iommu/amd/init.c

388

BUG_ON(iommu->mmio_base == NULL);

drivers/iommu/amd/init.c

3931

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

3933

for_each_iommu(iommu)

drivers/iommu/amd/init.c

3935

return iommu;

drivers/iommu/amd/init.c

3948

struct amd_iommu *iommu = get_amd_iommu(idx);

drivers/iommu/amd/init.c

395

memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,

drivers/iommu/amd/init.c

3950

if (iommu)

drivers/iommu/amd/init.c

3951

return iommu->max_banks;

drivers/iommu/amd/init.c

3963

struct amd_iommu *iommu = get_amd_iommu(idx);

drivers/iommu/amd/init.c

3965

if (iommu)

drivers/iommu/amd/init.c

3966

return iommu->max_counters;

drivers/iommu/amd/init.c

3971

static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,

drivers/iommu/amd/init.c

3982

if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))

drivers/iommu/amd/init.c

3988

max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |

drivers/iommu/amd/init.c

3989

(iommu->max_counters << 8) | 0x28);

drivers/iommu/amd/init.c

399

static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift)

drivers/iommu/amd/init.c

3997

writel((u32)val, iommu->mmio_base + offset);

drivers/iommu/amd/init.c

3998

writel((val >> 32), iommu->mmio_base + offset + 4);

drivers/iommu/amd/init.c

4000

*value = readl(iommu->mmio_base + offset + 4);

drivers/iommu/amd/init.c

4002

*value |= readl(iommu->mmio_base + offset);

drivers/iommu/amd/init.c

4009

int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)

drivers/iommu/amd/init.c

4011

if (!iommu)

drivers/iommu/amd/init.c

4014

return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);

drivers/iommu/amd/init.c

4017

int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)

drivers/iommu/amd/init.c

4019

if (!iommu)

drivers/iommu/amd/init.c

4022

return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);

drivers/iommu/amd/init.c

403

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

407

writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/init.c

4083

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

4089

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

4090

ret = iommu_make_shared(iommu->evt_buf, amd_iommu_evtlog_size);

drivers/iommu/amd/init.c

4094

ret = iommu_make_shared(iommu->ppr_log, amd_iommu_pprlog_size);

drivers/iommu/amd/init.c

4098

ret = iommu_make_shared((void *)iommu->cmd_sem, PAGE_SIZE);

drivers/iommu/amd/init.c

411

void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)

drivers/iommu/amd/init.c

413

iommu_feature_set(iommu, 1ULL, 1ULL, bit);

drivers/iommu/amd/init.c

416

static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)

drivers/iommu/amd/init.c

418

iommu_feature_set(iommu, 0ULL, 1ULL, bit);

drivers/iommu/amd/init.c

422

static void iommu_enable(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

424

iommu_feature_enable(iommu, CONTROL_IOMMU_EN);

drivers/iommu/amd/init.c

427

static void iommu_disable(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

429

if (!iommu->mmio_base)

drivers/iommu/amd/init.c

433

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

436

iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);

drivers/iommu/amd/init.c

437

iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);

drivers/iommu/amd/init.c

440

iommu_feature_disable(iommu, CONTROL_GALOG_EN);

drivers/iommu/amd/init.c

441

iommu_feature_disable(iommu, CONTROL_GAINT_EN);

drivers/iommu/amd/init.c

444

iommu_feature_disable(iommu, CONTROL_PPRLOG_EN);

drivers/iommu/amd/init.c

445

iommu_feature_disable(iommu, CONTROL_PPRINT_EN);

drivers/iommu/amd/init.c

448

iommu_feature_disable(iommu, CONTROL_IOMMU_EN);

drivers/iommu/amd/init.c

451

iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);

drivers/iommu/amd/init.c

470

static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

472

if (iommu->mmio_base)

drivers/iommu/amd/init.c

473

iounmap(iommu->mmio_base);

drivers/iommu/amd/init.c

474

release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);

drivers/iommu/amd/init.c

722

static int __init alloc_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

724

iommu->cmd_buf = iommu_alloc_pages_sz(GFP_KERNEL, CMD_BUFFER_SIZE);

drivers/iommu/amd/init.c

726

return iommu->cmd_buf ? 0 : -ENOMEM;

drivers/iommu/amd/init.c

733

void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,

drivers/iommu/amd/init.c

739

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

745

iommu_feature_disable(iommu, cntrl_log);

drivers/iommu/amd/init.c

746

iommu_feature_disable(iommu, cntrl_intr);

drivers/iommu/amd/init.c

748

writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

750

iommu_feature_enable(iommu, cntrl_intr);

drivers/iommu/amd/init.c

751

iommu_feature_enable(iommu, cntrl_log);

drivers/iommu/amd/init.c

758

void amd_iommu_restart_event_logging(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

760

amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN,

drivers/iommu/amd/init.c

769

void amd_iommu_restart_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

771

amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN,

drivers/iommu/amd/init.c

780

static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

782

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

784

writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

drivers/iommu/amd/init.c

785

writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/init.c

786

iommu->cmd_buf_head = 0;

drivers/iommu/amd/init.c

787

iommu->cmd_buf_tail = 0;

drivers/iommu/amd/init.c

789

iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

796

static void iommu_enable_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

800

BUG_ON(iommu->cmd_buf == NULL);

drivers/iommu/amd/init.c

807

entry = iommu_virt_to_phys(iommu->cmd_buf);

drivers/iommu/amd/init.c

809

memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,

drivers/iommu/amd/init.c

813

amd_iommu_reset_cmd_buffer(iommu);

drivers/iommu/amd/init.c

819

static void iommu_disable_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

821

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

drivers/iommu/amd/init.c

824

static void __init free_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

826

iommu_free_pages(iommu->cmd_buf);

drivers/iommu/amd/init.c

829

void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,

drivers/iommu/amd/init.c

832

int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;

drivers/iommu/amd/init.c

851

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

853

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

854

iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,

drivers/iommu/amd/init.c

856

if (!iommu->evt_buf)

drivers/iommu/amd/init.c

865

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

868

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

869

BUG_ON(iommu->evt_buf == NULL);

drivers/iommu/amd/init.c

876

entry = iommu_virt_to_phys(iommu->evt_buf);

drivers/iommu/amd/init.c

880

memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,

drivers/iommu/amd/init.c

885

writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

drivers/iommu/amd/init.c

886

writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

drivers/iommu/amd/init.c

888

iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);

drivers/iommu/amd/init.c

895

static void iommu_disable_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

897

iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);

drivers/iommu/amd/init.c

900

static void __init free_event_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

902

iommu_free_pages(iommu->evt_buf);

drivers/iommu/amd/init.c

905

static void free_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

908

iommu_free_pages(iommu->ga_log);

drivers/iommu/amd/init.c

909

iommu_free_pages(iommu->ga_log_tail);

drivers/iommu/amd/init.c

914

static int iommu_ga_log_enable(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

919

if (!iommu->ga_log)

drivers/iommu/amd/init.c

922

entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;

drivers/iommu/amd/init.c

923

memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,

drivers/iommu/amd/init.c

925

entry = (iommu_virt_to_phys(iommu->ga_log_tail) &

drivers/iommu/amd/init.c

927

memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,

drivers/iommu/amd/init.c

929

writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

drivers/iommu/amd/init.c

930

writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);

drivers/iommu/amd/init.c

933

iommu_feature_enable(iommu, CONTROL_GAINT_EN);

drivers/iommu/amd/init.c

934

iommu_feature_enable(iommu, CONTROL_GALOG_EN);

drivers/iommu/amd/init.c

937

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/init.c

949

static int iommu_init_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

951

int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;

drivers/iommu/amd/init.c

956

iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE);

drivers/iommu/amd/init.c

957

if (!iommu->ga_log)

drivers/iommu/amd/init.c

960

iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8);

drivers/iommu/amd/init.c

961

if (!iommu->ga_log_tail)

drivers/iommu/amd/init.c

966

free_ga_log(iommu);

drivers/iommu/amd/init.c

971

static int __init alloc_cwwb_sem(struct amd_iommu *iommu)

drivers/iommu/amd/init.c

973

iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1);

drivers/iommu/amd/init.c

974

if (!iommu->cmd_sem)

drivers/iommu/amd/init.c

976

iommu->cmd_sem_paddr = iommu_virt_to_phys((void *)iommu->cmd_sem);

drivers/iommu/amd/init.c

982

struct amd_iommu *iommu;

drivers/iommu/amd/init.c

986

for_each_iommu(iommu) {

drivers/iommu/amd/init.c

987

paddr = readq(iommu->mmio_base + MMIO_EVT_BUF_OFFSET) & PM_ADDR_MASK;

drivers/iommu/amd/init.c

988

iommu->evt_buf = iommu_memremap(paddr, amd_iommu_evtlog_size);

drivers/iommu/amd/init.c

989

if (!iommu->evt_buf)

drivers/iommu/amd/init.c

996

static int __init remap_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1006

static void iommu_poll_events(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1010

head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1011

tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1014

iommu_print_event(iommu, iommu->evt_buf + head);

drivers/iommu/amd/iommu.c

1018

writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1042

static void iommu_poll_ga_log(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1046

if (iommu->ga_log == NULL)

drivers/iommu/amd/iommu.c

1049

head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1050

tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1056

raw = (u64 *)(iommu->ga_log + head);

drivers/iommu/amd/iommu.c

1063

writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1085

amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1091

dev_set_msi_domain(dev, iommu->ir_domain);

drivers/iommu/amd/iommu.c

1096

amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }

drivers/iommu/amd/iommu.c

1104

struct amd_iommu *iommu = (struct amd_iommu *) data;

drivers/iommu/amd/iommu.c

1105

u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/iommu.c

1110

writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/iommu.c

1114

iommu->index, evt_type);

drivers/iommu/amd/iommu.c

1115

int_handler(iommu);

drivers/iommu/amd/iommu.c

1119

overflow_handler(iommu);

drivers/iommu/amd/iommu.c

1134

status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

drivers/iommu/amd/iommu.c

1182

static void dump_command_buffer(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1188

head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

drivers/iommu/amd/iommu.c

1189

tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1195

cmd = (struct iommu_cmd *)(iommu->cmd_buf + i * sizeof(*cmd));

drivers/iommu/amd/iommu.c

1201

static int wait_on_sem(struct amd_iommu *iommu, u64 data)

drivers/iommu/amd/iommu.c

1209

while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 &&

drivers/iommu/amd/iommu.c

1218

iommu->pci_seg->id, PCI_BUS_NUM(iommu->devid),

drivers/iommu/amd/iommu.c

1219

PCI_SLOT(iommu->devid), PCI_FUNC(iommu->devid));

drivers/iommu/amd/iommu.c

1222

DO_ONCE_LITE(dump_command_buffer, iommu);

drivers/iommu/amd/iommu.c

1230

static void copy_cmd_to_buffer(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1237

tail = iommu->cmd_buf_tail;

drivers/iommu/amd/iommu.c

1238

target = iommu->cmd_buf + tail;

drivers/iommu/amd/iommu.c

1242

iommu->cmd_buf_tail = tail;

drivers/iommu/amd/iommu.c

1245

writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

drivers/iommu/amd/iommu.c

1249

struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1252

u64 paddr = iommu->cmd_sem_paddr;

drivers/iommu/amd/iommu.c

1380

static int __iommu_queue_command_sync(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1387

next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;

drivers/iommu/amd/iommu.c

1389

left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;

drivers/iommu/amd/iommu.c

1403

iommu->cmd_buf_head = readl(iommu->mmio_base +

drivers/iommu/amd/iommu.c

1409

copy_cmd_to_buffer(iommu, cmd);

drivers/iommu/amd/iommu.c

1412

iommu->need_sync = sync;

drivers/iommu/amd/iommu.c

1417

static int iommu_queue_command_sync(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

1424

raw_spin_lock_irqsave(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1425

ret = __iommu_queue_command_sync(iommu, cmd, sync);

drivers/iommu/amd/iommu.c

1426

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1431

static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)

drivers/iommu/amd/iommu.c

1433

return iommu_queue_command_sync(iommu, cmd, true);

drivers/iommu/amd/iommu.c

1436

static u64 get_cmdsem_val(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1438

lockdep_assert_held(&iommu->lock);

drivers/iommu/amd/iommu.c

1439

return ++iommu->cmd_sem_val;

drivers/iommu/amd/iommu.c

1446

static int iommu_completion_wait(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1453

raw_spin_lock_irqsave(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1455

if (!iommu->need_sync) {

drivers/iommu/amd/iommu.c

146

static void update_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data,

drivers/iommu/amd/iommu.c

1465

data = iommu->cmd_sem_val;

drivers/iommu/amd/iommu.c

1466

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1467

return wait_on_sem(iommu, data);

drivers/iommu/amd/iommu.c

1470

data = get_cmdsem_val(iommu);

drivers/iommu/amd/iommu.c

1471

build_completion_wait(&cmd, iommu, data);

drivers/iommu/amd/iommu.c

1473

ret = __iommu_queue_command_sync(iommu, &cmd, false);

drivers/iommu/amd/iommu.c

1474

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

1479

return wait_on_sem(iommu, data);

drivers/iommu/amd/iommu.c

1494

iommu_completion_wait(pdom_iommu_info->iommu);

drivers/iommu/amd/iommu.c

1497

static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

150

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/iommu.c

1503

return iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1506

static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

1510

ret = iommu_flush_dte(iommu, devid);

drivers/iommu/amd/iommu.c

1512

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1515

static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1518

u16 last_bdf = iommu->pci_seg->last_bdf;

drivers/iommu/amd/iommu.c

1521

iommu_flush_dte(iommu, devid);

drivers/iommu/amd/iommu.c

1523

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1530

static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1533

u16 last_bdf = iommu->pci_seg->last_bdf;

drivers/iommu/amd/iommu.c

1540

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1543

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1546

static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)

drivers/iommu/amd/iommu.c

1553

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1555

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1567

struct amd_iommu *iommu = container_of(aviommu->core.iommu_dev,

drivers/iommu/amd/iommu.c

1568

struct amd_iommu, iommu);

drivers/iommu/amd/iommu.c

1575

iommu->devid, gdom_info->hdom_id);

drivers/iommu/amd/iommu.c

1578

ret |= iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1585

static void amd_iommu_flush_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

159

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1591

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1592

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1595

static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

1601

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1604

static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1607

u16 last_bdf = iommu->pci_seg->last_bdf;

drivers/iommu/amd/iommu.c

1609

if (iommu->irtcachedis_enabled)

drivers/iommu/amd/iommu.c

1613

iommu_flush_irt(iommu, devid);

drivers/iommu/amd/iommu.c

1615

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1618

void amd_iommu_flush_all_caches(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

1621

amd_iommu_flush_all(iommu);

drivers/iommu/amd/iommu.c

1623

amd_iommu_flush_dte_all(iommu);

drivers/iommu/amd/iommu.c

1624

amd_iommu_flush_irt_all(iommu);

drivers/iommu/amd/iommu.c

1625

amd_iommu_flush_tlb_all(iommu);

drivers/iommu/amd/iommu.c

1635

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

164

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1642

return iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1647

struct amd_iommu *iommu = data;

drivers/iommu/amd/iommu.c

1649

return iommu_flush_dte(iommu, alias);

drivers/iommu/amd/iommu.c

1657

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

1668

device_flush_dte_alias, iommu);

drivers/iommu/amd/iommu.c

1670

ret = iommu_flush_dte(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1674

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

1677

ret = iommu_flush_dte(iommu, alias);

drivers/iommu/amd/iommu.c

1700

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/iommu.c

1707

ret |= iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

172

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1731

ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd);

drivers/iommu/amd/iommu.c

181

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1833

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/iommu.c

1839

iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

1844

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

1856

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

1860

iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

1865

return iommu_queue_command(iommu, &cmd);

drivers/iommu/amd/iommu.c

194

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

1962

struct amd_iommu *iommu, int pasids)

drivers/iommu/amd/iommu.c

1965

int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;

drivers/iommu/amd/iommu.c

199

iommu_flush_dte_sync(iommu, dev_data->devid);

drivers/iommu/amd/iommu.c

212

void amd_iommu_update_dte(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

2159

static void set_dte_entry(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

216

update_dte256(iommu, dev_data, new);

drivers/iommu/amd/iommu.c

2167

struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];

drivers/iommu/amd/iommu.c

217

clone_aliases(iommu, dev_data->dev);

drivers/iommu/amd/iommu.c

2182

amd_iommu_update_dte(iommu, dev_data, &new);

drivers/iommu/amd/iommu.c

219

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

2190

amd_iommu_flush_tlb_domid(iommu, old_domid);

drivers/iommu/amd/iommu.c

2197

static void clear_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data)

drivers/iommu/amd/iommu.c

2202

amd_iommu_update_dte(iommu, dev_data, &new);

drivers/iommu/amd/iommu.c

2208

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

drivers/iommu/amd/iommu.c

2211

set_dte_entry(iommu, dev_data, 0, 0);

drivers/iommu/amd/iommu.c

2213

clear_dte_entry(iommu, dev_data);

drivers/iommu/amd/iommu.c

222

static void get_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data,

drivers/iommu/amd/iommu.c

2223

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

2239

ret = setup_gcr3_table(&dev_data->gcr3_info, iommu,

drivers/iommu/amd/iommu.c

227

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/iommu.c

2270

static int pdom_attach_iommu(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

2279

pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);

drivers/iommu/amd/iommu.c

2291

pdom_iommu_info->iommu = iommu;

drivers/iommu/amd/iommu.c

2294

curr = xa_cmpxchg(&pdom->iommu_array, iommu->index,

drivers/iommu/amd/iommu.c

2307

static void pdom_detach_iommu(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

2315

pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);

drivers/iommu/amd/iommu.c

2323

xa_erase(&pdom->iommu_array, iommu->index);

drivers/iommu/amd/iommu.c

2338

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

2351

ret = pdom_attach_iommu(iommu, domain);

drivers/iommu/amd/iommu.c

2359

pdom_detach_iommu(iommu, domain);

drivers/iommu/amd/iommu.c

2373

if (amd_iommu_iopf_add_device(iommu, dev_data))

drivers/iommu/amd/iommu.c

2400

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

2418

amd_iommu_iopf_remove_device(iommu, dev_data);

drivers/iommu/amd/iommu.c

2441

pdom_detach_iommu(iommu, domain);

drivers/iommu/amd/iommu.c

2450

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

2457

iommu = rlookup_amd_iommu(dev);

drivers/iommu/amd/iommu.c

2458

if (!iommu)

drivers/iommu/amd/iommu.c

2462

if (!iommu->iommu.ops)

drivers/iommu/amd/iommu.c

2466

return &iommu->iommu;

drivers/iommu/amd/iommu.c

2468

ret = iommu_init_device(iommu, dev);

drivers/iommu/amd/iommu.c

2472

iommu_ignore_device(iommu, dev);

drivers/iommu/amd/iommu.c

2476

amd_iommu_set_pci_msi_domain(dev, iommu);

drivers/iommu/amd/iommu.c

2477

iommu_dev = &iommu->iommu;

drivers/iommu/amd/iommu.c

2486

dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,

drivers/iommu/amd/iommu.c

2497

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

2571

static bool amd_iommu_hd_support(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

2576

return iommu && (iommu->features & FEATURE_HDSUP);

drivers/iommu/amd/iommu.c

2582

container_of(iommupt, struct protection_domain, iommu);

drivers/iommu/amd/iommu.c

2594

container_of(iommu_table, struct protection_domain, iommu);

drivers/iommu/amd/iommu.c

2601

struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev);

drivers/iommu/amd/iommu.c

2604

set_dte_entry(iommu, dev_data, top_paddr, top_level);

drivers/iommu/amd/iommu.c

2605

clone_aliases(iommu, dev_data->dev);

drivers/iommu/amd/iommu.c

2694

domain->iommu.driver_ops = &amd_hw_driver_ops_v1;

drivers/iommu/amd/iommu.c

2695

domain->iommu.nid = dev_to_node(dev);

drivers/iommu/amd/iommu.c

2775

domain->iommu.nid = dev_to_node(dev);

drivers/iommu/amd/iommu.c

2822

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

2839

!amd_iommu_hd_support(iommu))

drivers/iommu/amd/iommu.c

2878

pt_iommu_deinit(&domain->iommu);

drivers/iommu/amd/iommu.c

2953

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

2969

if (dom->dirty_ops && !amd_iommu_hd_support(iommu))

drivers/iommu/amd/iommu.c

3001

struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

3003

return amd_iommu_hd_support(iommu);

drivers/iommu/amd/iommu.c

3025

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

3037

iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/iommu.c

3038

dte = &get_dev_table(iommu)[dev_data->devid];

drivers/iommu/amd/iommu.c

3064

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

3073

iommu = get_amd_iommu_from_dev(dev);

drivers/iommu/amd/iommu.c

3074

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

312

struct dev_table_entry *get_dev_table(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

315

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3193

struct amd_iommu *iommu = data;

drivers/iommu/amd/iommu.c

3196

ret = __iommu_queue_command_sync(iommu, &cmd, true);

drivers/iommu/amd/iommu.c

3200

static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3207

struct iommu_dev_data *dev_data = search_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

3209

if (iommu->irtcachedis_enabled)

drivers/iommu/amd/iommu.c

3215

raw_spin_lock_irqsave(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

3216

data = get_cmdsem_val(iommu);

drivers/iommu/amd/iommu.c

3217

build_completion_wait(&cmd, iommu, data);

drivers/iommu/amd/iommu.c

3220

ret = pci_for_each_dma_alias(pdev, iommu_flush_dev_irt, iommu);

drivers/iommu/amd/iommu.c

3222

ret = iommu_flush_dev_irt(NULL, devid, iommu);

drivers/iommu/amd/iommu.c

3226

ret = __iommu_queue_command_sync(iommu, &cmd, false);

drivers/iommu/amd/iommu.c

3229

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

3231

wait_on_sem(iommu, data);

drivers/iommu/amd/iommu.c

3235

raw_spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/amd/iommu.c

3245

static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,

drivers/iommu/amd/iommu.c

3249

struct dev_table_entry *dte = &get_dev_table(iommu)[devid];

drivers/iommu/amd/iommu.c

3250

struct iommu_dev_data *dev_data = search_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

3267

static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3270

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3304

static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,

drivers/iommu/amd/iommu.c

3307

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3310

set_dte_irq_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3311

iommu_flush_dte(iommu, devid);

drivers/iommu/amd/iommu.c

3319

struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);

drivers/iommu/amd/iommu.c

3321

if (!iommu)

drivers/iommu/amd/iommu.c

3324

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3326

set_dte_irq_entry(iommu, alias, table);

drivers/iommu/amd/iommu.c

3340

static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

3348

int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;

drivers/iommu/amd/iommu.c

3353

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3361

set_remap_table_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3379

set_remap_table_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3390

set_remap_table_entry(iommu, devid, table);

drivers/iommu/amd/iommu.c

3393

set_remap_table_entry(iommu, alias, table);

drivers/iommu/amd/iommu.c

3396

iommu_completion_wait(iommu);

drivers/iommu/amd/iommu.c

3408

static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,

drivers/iommu/amd/iommu.c

3416

table = alloc_irq_table(iommu, devid, pdev, max_irqs);

drivers/iommu/amd/iommu.c

342

void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3428

if (!iommu->irte_ops->is_allocated(table, index)) {

drivers/iommu/amd/iommu.c

3438

iommu->irte_ops->set_allocated(table, index - c + 1);

drivers/iommu/amd/iommu.c

344

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3455

static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,

drivers/iommu/amd/iommu.c

346

pci_seg->rlookup_table[devid] = iommu;

drivers/iommu/amd/iommu.c

3463

table = get_irq_table(iommu, devid);

drivers/iommu/amd/iommu.c

3486

static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,

drivers/iommu/amd/iommu.c

3491

ret = __modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3495

iommu_flush_irt_and_complete(iommu, devid);

drivers/iommu/amd/iommu.c

3500

static int modify_irte(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

3506

table = get_irq_table(iommu, devid);

drivers/iommu/amd/iommu.c

3514

iommu_flush_irt_and_complete(iommu, devid);

drivers/iommu/amd/iommu.c

3519

static void free_irte(struct amd_iommu *iommu, u16 devid, int index)

drivers/iommu/amd/iommu.c

3524

table = get_irq_table(iommu, devid);

drivers/iommu/amd/iommu.c

3529

iommu->irte_ops->clear_allocated(table, index);

drivers/iommu/amd/iommu.c

3532

iommu_flush_irt_and_complete(iommu, devid);

drivers/iommu/amd/iommu.c

3565

static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3570

modify_irte(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3573

static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3578

modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3581

static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3586

modify_irte(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3589

static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

drivers/iommu/amd/iommu.c

3594

modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3597

static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,

drivers/iommu/amd/iommu.c

3604

modify_irte(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3607

static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,

drivers/iommu/amd/iommu.c

3618

modify_irte_ga(iommu, devid, index, irte);

drivers/iommu/amd/iommu.c

3714

struct amd_iommu *iommu = data->iommu;

drivers/iommu/amd/iommu.c

3716

if (!iommu)

drivers/iommu/amd/iommu.c

3721

iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED,

drivers/iommu/amd/iommu.c

374

static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3765

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

377

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

3783

iommu = __rlookup_amd_iommu(seg, devid);

drivers/iommu/amd/iommu.c

3784

if (!iommu)

drivers/iommu/amd/iommu.c

3787

dev_data = search_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

3797

table = alloc_irq_table(iommu, devid, NULL, max_irqs);

drivers/iommu/amd/iommu.c

3806

iommu->irte_ops->set_allocated(table, i);

drivers/iommu/amd/iommu.c

3817

index = alloc_irq_index(iommu, devid, nr_irqs, align,

drivers/iommu/amd/iommu.c

3821

index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL,

drivers/iommu/amd/iommu.c

3853

data->iommu = iommu;

drivers/iommu/amd/iommu.c

3869

free_irte(iommu, devid, index + i);

drivers/iommu/amd/iommu.c

3888

free_irte(data->iommu, irte_info->devid, irte_info->index);

drivers/iommu/amd/iommu.c

3896

static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

3906

struct amd_iommu *iommu = data->iommu;

drivers/iommu/amd/iommu.c

3909

if (!iommu)

drivers/iommu/amd/iommu.c

3912

iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,

drivers/iommu/amd/iommu.c

3914

amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);

drivers/iommu/amd/iommu.c

392

struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

3923

struct amd_iommu *iommu = data->iommu;

drivers/iommu/amd/iommu.c

3925

if (iommu)

drivers/iommu/amd/iommu.c

3926

iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,

drivers/iommu/amd/iommu.c

3933

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

3946

iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));

drivers/iommu/amd/iommu.c

3948

return iommu && iommu->ir_domain == d;

drivers/iommu/amd/iommu.c

396

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

4003

if (!ir_data->iommu)

drivers/iommu/amd/iommu.c

4008

return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

drivers/iommu/amd/iommu.c

4038

return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

drivers/iommu/amd/iommu.c

4070

return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

drivers/iommu/amd/iommu.c

4086

if (ir_data->iommu == NULL)

drivers/iommu/amd/iommu.c

4089

dev_data = search_dev_data(ir_data->iommu, irte_info->devid);

drivers/iommu/amd/iommu.c

4119

static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

4129

iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,

drivers/iommu/amd/iommu.c

413

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

4141

struct amd_iommu *iommu = ir_data->iommu;

drivers/iommu/amd/iommu.c

4144

if (!iommu)

drivers/iommu/amd/iommu.c

4151

amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg);

drivers/iommu/amd/iommu.c

4185

int amd_iommu_create_irq_domain(struct amd_iommu *iommu)

drivers/iommu/amd/iommu.c

4188

.fwnode = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index),

drivers/iommu/amd/iommu.c

4191

.host_data = iommu,

drivers/iommu/amd/iommu.c

4198

iommu->ir_domain = msi_create_parent_irq_domain(&info, &amdvi_msi_parent_ops);

drivers/iommu/amd/iommu.c

4199

if (!iommu->ir_domain) {

drivers/iommu/amd/iommu.c

422

iommu = rlookup_amd_iommu(&pdev->dev);

drivers/iommu/amd/iommu.c

423

if (!iommu)

drivers/iommu/amd/iommu.c

433

get_dte256(iommu, dev_data, &new);

drivers/iommu/amd/iommu.c

436

alias_data = find_dev_data(iommu, alias);

drivers/iommu/amd/iommu.c

442

update_dte256(iommu, alias_data, &new);

drivers/iommu/amd/iommu.c

444

amd_iommu_set_rlookup_table(iommu, alias);

drivers/iommu/amd/iommu.c

449

static void clone_aliases(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

462

clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], pdev);

drivers/iommu/amd/iommu.c

467

static void setup_aliases(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

470

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

486

clone_aliases(iommu, dev);

drivers/iommu/amd/iommu.c

489

static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

493

dev_data = search_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

496

dev_data = alloc_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

500

if (translation_pre_enabled(iommu))

drivers/iommu/amd/iommu.c

678

struct amd_iommu *iommu;

drivers/iommu/amd/iommu.c

689

iommu = rlookup_amd_iommu(dev);

drivers/iommu/amd/iommu.c

690

if (!iommu)

drivers/iommu/amd/iommu.c

694

pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

701

static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

714

dev_data = find_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

725

setup_aliases(iommu, dev);

drivers/iommu/amd/iommu.c

741

static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)

drivers/iommu/amd/iommu.c

743

struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

drivers/iommu/amd/iommu.c

744

struct dev_table_entry *dev_table = get_dev_table(iommu);

drivers/iommu/amd/iommu.c

75

static void set_dte_entry(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

755

setup_aliases(iommu, dev);

drivers/iommu/amd/iommu.c

765

static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)

drivers/iommu/amd/iommu.c

769

struct iommu_dev_data *dev_data = find_dev_data(iommu, devid);

drivers/iommu/amd/iommu.c

771

get_dte256(iommu, dev_data, &dte);

drivers/iommu/amd/iommu.c

786

static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event)

drivers/iommu/amd/iommu.c

798

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

drivers/iommu/amd/iommu.c

810

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

818

static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event)

drivers/iommu/amd/iommu.c

831

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

drivers/iommu/amd/iommu.c

84

static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/iommu.c

843

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

857

static void amd_iommu_report_page_fault(struct amd_iommu *iommu,

drivers/iommu/amd/iommu.c

86

static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid);

drivers/iommu/amd/iommu.c

864

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

drivers/iommu/amd/iommu.c

880

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),

drivers/iommu/amd/iommu.c

899

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

908

static void iommu_print_event(struct amd_iommu *iommu, void *__evt)

drivers/iommu/amd/iommu.c

91

static void clone_aliases(struct amd_iommu *iommu, struct device *dev);

drivers/iommu/amd/iommu.c

910

struct device *dev = iommu->iommu.dev;

drivers/iommu/amd/iommu.c

924

ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);

drivers/iommu/amd/iommu.c

93

static int iommu_completion_wait(struct amd_iommu *iommu);

drivers/iommu/amd/iommu.c

937

amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);

drivers/iommu/amd/iommu.c

944

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

947

dump_dte_entry(iommu, devid);

drivers/iommu/amd/iommu.c

952

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

957

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

970

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

975

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/iommu.c

979

amd_iommu_report_rmp_fault(iommu, event);

drivers/iommu/amd/iommu.c

982

amd_iommu_report_rmp_hw_error(iommu, event);

drivers/iommu/amd/iommu.c

988

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/nested.c

190

static void set_dte_nested(struct amd_iommu *iommu, struct iommu_domain *dom,

drivers/iommu/amd/nested.c

241

struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);

drivers/iommu/amd/nested.c

253

set_dte_nested(iommu, dom, dev_data, &new);

drivers/iommu/amd/nested.c

255

amd_iommu_update_dte(iommu, dev_data, &new);

drivers/iommu/amd/ppr.c

107

static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)

drivers/iommu/amd/ppr.c

119

pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,

drivers/iommu/amd/ppr.c

124

if (!ppr_is_valid(iommu, raw))

drivers/iommu/amd/ppr.c

164

void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

168

if (iommu->ppr_log == NULL)

drivers/iommu/amd/ppr.c

171

head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

drivers/iommu/amd/ppr.c

172

tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

drivers/iommu/amd/ppr.c

179

raw = (u64 *)(iommu->ppr_log + head);

drivers/iommu/amd/ppr.c

20

int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

207

writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

drivers/iommu/amd/ppr.c

210

iommu_call_iopf_notifier(iommu, entry);

drivers/iommu/amd/ppr.c

22

iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,

drivers/iommu/amd/ppr.c

220

int amd_iommu_iopf_init(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

224

if (iommu->iopf_queue)

drivers/iommu/amd/ppr.c

227

snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), "amdvi-%#x",

drivers/iommu/amd/ppr.c

228

PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, iommu->devid));

drivers/iommu/amd/ppr.c

230

iommu->iopf_queue = iopf_queue_alloc(iommu->iopfq_name);

drivers/iommu/amd/ppr.c

231

if (!iommu->iopf_queue)

drivers/iommu/amd/ppr.c

238

void amd_iommu_iopf_uninit(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

24

return iommu->ppr_log ? 0 : -ENOMEM;

drivers/iommu/amd/ppr.c

240

iopf_queue_free(iommu->iopf_queue);

drivers/iommu/amd/ppr.c

241

iommu->iopf_queue = NULL;

drivers/iommu/amd/ppr.c

250

int amd_iommu_iopf_add_device(struct amd_iommu *iommu,

drivers/iommu/amd/ppr.c

258

if (!iommu->iopf_queue)

drivers/iommu/amd/ppr.c

261

ret = iopf_queue_add_device(iommu->iopf_queue, dev_data->dev);

drivers/iommu/amd/ppr.c

27

void amd_iommu_enable_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

270

void amd_iommu_iopf_remove_device(struct amd_iommu *iommu,

drivers/iommu/amd/ppr.c

273

iopf_queue_remove_device(iommu->iopf_queue, dev_data->dev);

drivers/iommu/amd/ppr.c

31

if (iommu->ppr_log == NULL)

drivers/iommu/amd/ppr.c

34

iommu_feature_enable(iommu, CONTROL_PPR_EN);

drivers/iommu/amd/ppr.c

36

entry = iommu_virt_to_phys(iommu->ppr_log);

drivers/iommu/amd/ppr.c

40

memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,

drivers/iommu/amd/ppr.c

44

writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

drivers/iommu/amd/ppr.c

45

writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

drivers/iommu/amd/ppr.c

47

iommu_feature_enable(iommu, CONTROL_PPRINT_EN);

drivers/iommu/amd/ppr.c

48

iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);

drivers/iommu/amd/ppr.c

51

void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

53

iommu_free_pages(iommu->ppr_log);

drivers/iommu/amd/ppr.c

60

void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)

drivers/iommu/amd/ppr.c

62

amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN,

drivers/iommu/amd/ppr.c

83

static bool ppr_is_valid(struct amd_iommu *iommu, u64 *raw)

drivers/iommu/amd/ppr.c

85

struct device *dev = iommu->iommu.dev;

drivers/iommu/amd/ppr.c

91

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/amd/ppr.c

99

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

drivers/iommu/apple-dart.c

1184

ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",

drivers/iommu/apple-dart.c

1189

ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);

drivers/iommu/apple-dart.c

1201

iommu_device_sysfs_remove(&dart->iommu);

drivers/iommu/apple-dart.c

1217

iommu_device_unregister(&dart->iommu);

drivers/iommu/apple-dart.c

1218

iommu_device_sysfs_remove(&dart->iommu);

drivers/iommu/apple-dart.c

227

struct iommu_device iommu;

drivers/iommu/apple-dart.c

757

return &cfg->stream_maps[0].dart->iommu;

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c

466

container_of(viommu->iommu_dev, struct arm_smmu_device, iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

4203

return &smmu->iommu;

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

5076

smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

5489

ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

5494

ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

5503

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

5515

iommu_device_unregister(&smmu->iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

5516

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h

956

struct iommu_device iommu;

drivers/iommu/arm/arm-smmu/arm-smmu.c

1497

return &smmu->iommu;

drivers/iommu/arm/arm-smmu/arm-smmu.c

2235

err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,

drivers/iommu/arm/arm-smmu/arm-smmu.c

2240

err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,

drivers/iommu/arm/arm-smmu/arm-smmu.c

2243

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu/arm-smmu.c

2286

iommu_device_unregister(&smmu->iommu);

drivers/iommu/arm/arm-smmu/arm-smmu.c

2287

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/arm/arm-smmu/arm-smmu.h

342

struct iommu_device iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

228

if (qcom_domain->iommu)

drivers/iommu/arm/arm-smmu/qcom_iommu.c

239

qcom_domain->iommu = qcom_iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

317

qcom_domain->iommu = NULL;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

347

if (qcom_domain->iommu) {

drivers/iommu/arm/arm-smmu/qcom_iommu.c

354

pm_runtime_get_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

356

pm_runtime_put_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

385

if (qcom_domain->iommu != qcom_iommu)

drivers/iommu/arm/arm-smmu/qcom_iommu.c

404

if (WARN_ON(!qcom_domain->iommu))

drivers/iommu/arm/arm-smmu/qcom_iommu.c

464

pm_runtime_get_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

468

pm_runtime_put_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

48

struct iommu_device iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

481

pm_runtime_get_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

483

pm_runtime_put_sync(qcom_domain->iommu->dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

546

return &qcom_iommu->iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

71

struct qcom_iommu_dev *iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

84

struct qcom_iommu_dev *qcom_iommu = d->iommu;

drivers/iommu/arm/arm-smmu/qcom_iommu.c

848

ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,

drivers/iommu/arm/arm-smmu/qcom_iommu.c

855

ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

880

iommu_device_sysfs_remove(&qcom_iommu->iommu);

drivers/iommu/arm/arm-smmu/qcom_iommu.c

881

iommu_device_unregister(&qcom_iommu->iommu);

drivers/iommu/dma-iommu.c

2155

dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac;

drivers/iommu/dma-iommu.c

641

if (dev->iommu->shadow_on_flush) {

drivers/iommu/dma-iommu.c

791

if (dma_limit > DMA_BIT_MASK(32) && dev->iommu->pci_32bit_workaround) {

drivers/iommu/dma-iommu.c

797

dev->iommu->pci_32bit_workaround = false;

drivers/iommu/exynos-iommu.c

1426

return &data->iommu;

drivers/iommu/exynos-iommu.c

304

struct iommu_device iommu; /* IOMMU core handle */

drivers/iommu/exynos-iommu.c

780

ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL,

drivers/iommu/exynos-iommu.c

816

ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev);

drivers/iommu/exynos-iommu.c

823

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/generic_pt/fmt/amdv1.h

331

return &container_of(iommu_table, struct pt_iommu_amdv1, iommu)

drivers/iommu/generic_pt/fmt/amdv1.h

337

return &container_of(common, struct pt_iommu_amdv1, amdpt.common)->iommu;

drivers/iommu/generic_pt/fmt/riscv.h

220

return &container_of(iommu_table, struct pt_iommu_table, iommu)

drivers/iommu/generic_pt/fmt/riscv.h

227

->iommu;

drivers/iommu/generic_pt/fmt/vtdss.h

205

return &container_of(iommu_table, struct pt_iommu_table, iommu)

drivers/iommu/generic_pt/fmt/vtdss.h

212

->iommu;

drivers/iommu/generic_pt/fmt/x86_64.h

206

return &container_of(iommu_table, struct pt_iommu_table, iommu)

drivers/iommu/generic_pt/fmt/x86_64.h

213

->iommu;

drivers/iommu/generic_pt/iommu_pt.h

1275

struct pt_iommu *iommu_table = &fmt_table->iommu;

drivers/iommu/generic_pt/iommu_pt.h

1278

static_assert(offsetof(struct pt_iommu_table, iommu.domain) == 0);

drivers/iommu/generic_pt/iommu_pt.h

1279

memset_after(fmt_table, 0, iommu.domain);

drivers/iommu/generic_pt/iommu_pt.h

1293

struct pt_iommu *iommu_table = &fmt_table->iommu;

drivers/iommu/generic_pt/iommu_pt.h

1354

struct pt_iommu *iommu_table = &fmt_table->iommu;

drivers/iommu/generic_pt/kunit_generic_pt.h

829

pt_iommu_deinit(priv->iommu);

drivers/iommu/generic_pt/kunit_iommu.h

140

priv->fmt_table.iommu.nid = NUMA_NO_NODE;

drivers/iommu/generic_pt/kunit_iommu.h

141

priv->fmt_table.iommu.driver_ops = &pt_kunit_driver_ops;

drivers/iommu/generic_pt/kunit_iommu.h

142

priv->fmt_table.iommu.iommu_device = priv->dummy_dev;

drivers/iommu/generic_pt/kunit_iommu.h

153

priv->iommu = &priv->fmt_table.iommu;

drivers/iommu/generic_pt/kunit_iommu.h

154

priv->common = common_from_iommu(&priv->fmt_table.iommu);

drivers/iommu/generic_pt/kunit_iommu.h

155

priv->iommu->ops->get_info(priv->iommu, &priv->info);

drivers/iommu/generic_pt/kunit_iommu.h

65

struct pt_iommu *iommu;

drivers/iommu/generic_pt/kunit_iommu.h

77

PT_IOMMU_CHECK_DOMAIN(struct kunit_iommu_priv, fmt_table.iommu, domain);

drivers/iommu/generic_pt/kunit_iommu.h

99

iommu_table, struct kunit_iommu_priv, fmt_table.iommu);

drivers/iommu/generic_pt/kunit_iommu_pt.h

468

pt_iommu_deinit(priv->iommu);

drivers/iommu/intel/cache.c

104

if (cache_tage_match(tag, did, iommu, dev, pasid, type)) {

drivers/iommu/intel/cache.c

202

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/cache.c

211

return domain_id_iommu(domain, iommu);

drivers/iommu/intel/cache.c

24

struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/cache.c

282

static void qi_batch_flush_descs(struct intel_iommu *iommu, struct qi_batch *batch)

drivers/iommu/intel/cache.c

284

if (!iommu || !batch->index)

drivers/iommu/intel/cache.c

287

qi_submit_sync(iommu, batch->descs, batch->index, 0);

drivers/iommu/intel/cache.c

293

static void qi_batch_increment_index(struct intel_iommu *iommu, struct qi_batch *batch)

drivers/iommu/intel/cache.c

296

qi_batch_flush_descs(iommu, batch);

drivers/iommu/intel/cache.c

299

static void qi_batch_add_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/cache.c

303

qi_desc_iotlb(iommu, did, addr, size_order, type, &batch->descs[batch->index]);

drivers/iommu/intel/cache.c

304

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

307

static void qi_batch_add_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/cache.c

315

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/cache.c

319

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

322

static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did,

drivers/iommu/intel/cache.c

326

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

329

static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,

drivers/iommu/intel/cache.c

335

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

338

static void qi_batch_add_pasid_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/cache.c

34

return tag->iommu == iommu;

drivers/iommu/intel/cache.c

347

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/cache.c

352

qi_batch_increment_index(iommu, batch);

drivers/iommu/intel/cache.c

365

struct intel_iommu *iommu = tag->iommu;

drivers/iommu/intel/cache.c

370

qi_batch_add_piotlb_all(iommu, tag->domain_id,

drivers/iommu/intel/cache.c

373

qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid,

drivers/iommu/intel/cache.c

382

if (!cap_pgsel_inv(iommu->cap) ||

drivers/iommu/intel/cache.c

383

mask > cap_max_amask_val(iommu->cap)) {

drivers/iommu/intel/cache.c

390

if (ecap_qis(iommu->ecap))

drivers/iommu/intel/cache.c

391

qi_batch_add_iotlb(iommu, tag->domain_id, addr | ih, mask, type,

drivers/iommu/intel/cache.c

394

__iommu_flush_iotlb(iommu, tag->domain_id, addr | ih, mask, type);

drivers/iommu/intel/cache.c

400

struct intel_iommu *iommu = tag->iommu;

drivers/iommu/intel/cache.c

408

qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep,

drivers/iommu/intel/cache.c

411

qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep,

drivers/iommu/intel/cache.c

416

qi_batch_add_pasid_dev_iotlb(iommu, sid, info->pfsid, tag->pasid,

drivers/iommu/intel/cache.c

419

qi_batch_add_pasid_dev_iotlb(iommu, sid, info->pfsid, tag->pasid,

drivers/iommu/intel/cache.c

431

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/cache.c

440

if (iommu && iommu != tag->iommu)

drivers/iommu/intel/cache.c

441

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

442

iommu = tag->iommu;

drivers/iommu/intel/cache.c

467

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

47

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/cache.c

494

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/cache.c

503

if (iommu && iommu != tag->iommu)

drivers/iommu/intel/cache.c

504

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

505

iommu = tag->iommu;

drivers/iommu/intel/cache.c

507

if (!cap_caching_mode(iommu->cap) ||

drivers/iommu/intel/cache.c

509

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/cache.c

519

qi_batch_flush_descs(iommu, domain->qi_batch);

drivers/iommu/intel/cache.c

57

tag->iommu = iommu;

drivers/iommu/intel/cache.c

65

tag->dev = iommu->iommu.dev;

drivers/iommu/intel/cache.c

70

if (cache_tage_match(temp, did, iommu, dev, pasid, type)) {

drivers/iommu/intel/cache.c

77

if (temp->iommu == iommu)

drivers/iommu/intel/cache.c

98

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/debugfs.c

114

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

120

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

128

iommu->name, drhd->reg_base_addr);

drivers/iommu/intel/debugfs.c

134

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/debugfs.c

136

value = readl(iommu->reg + iommu_regs_32[i].offset);

drivers/iommu/intel/debugfs.c

142

value = readq(iommu->reg + iommu_regs_64[i].offset);

drivers/iommu/intel/debugfs.c

147

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/debugfs.c

215

static void ctx_tbl_walk(struct seq_file *m, struct intel_iommu *iommu, u16 bus)

drivers/iommu/intel/debugfs.c

237

context = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/debugfs.c

246

tbl_wlk.rt_entry = &iommu->root_entry[bus];

drivers/iommu/intel/debugfs.c

250

if (readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT) {

drivers/iommu/intel/debugfs.c

261

static void root_tbl_walk(struct seq_file *m, struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

265

spin_lock(&iommu->lock);

drivers/iommu/intel/debugfs.c

266

seq_printf(m, "IOMMU %s: Root Table Address: 0x%llx\n", iommu->name,

drivers/iommu/intel/debugfs.c

267

(u64)virt_to_phys(iommu->root_entry));

drivers/iommu/intel/debugfs.c

276

ctx_tbl_walk(m, iommu, bus);

drivers/iommu/intel/debugfs.c

277

spin_unlock(&iommu->lock);

drivers/iommu/intel/debugfs.c

283

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

287

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

288

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/debugfs.c

291

iommu->name);

drivers/iommu/intel/debugfs.c

294

root_tbl_walk(m, iommu);

drivers/iommu/intel/debugfs.c

351

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

359

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

364

if (seg != iommu->segment)

drivers/iommu/intel/debugfs.c

367

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/debugfs.c

370

iommu->name);

drivers/iommu/intel/debugfs.c

373

if (readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT)

drivers/iommu/intel/debugfs.c

387

spin_lock(&iommu->lock);

drivers/iommu/intel/debugfs.c

389

context = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/debugfs.c

454

iommu->segment, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));

drivers/iommu/intel/debugfs.c

467

spin_unlock(&iommu->lock);

drivers/iommu/intel/debugfs.c

494

struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

496

int index, shift = qi_shift(iommu);

drivers/iommu/intel/debugfs.c

500

if (ecap_smts(iommu->ecap))

drivers/iommu/intel/debugfs.c

507

desc = iommu->qi->desc + offset;

drivers/iommu/intel/debugfs.c

508

if (ecap_smts(iommu->ecap))

drivers/iommu/intel/debugfs.c

512

iommu->qi->desc_status[index]);

drivers/iommu/intel/debugfs.c

516

iommu->qi->desc_status[index]);

drivers/iommu/intel/debugfs.c

523

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

529

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

530

qi = iommu->qi;

drivers/iommu/intel/debugfs.c

531

shift = qi_shift(iommu);

drivers/iommu/intel/debugfs.c

533

if (!qi || !ecap_qis(iommu->ecap))

drivers/iommu/intel/debugfs.c

536

seq_printf(m, "Invalidation queue on IOMMU: %s\n", iommu->name);

drivers/iommu/intel/debugfs.c

541

readq(iommu->reg + DMAR_IQH_REG) >> shift,

drivers/iommu/intel/debugfs.c

542

readq(iommu->reg + DMAR_IQT_REG) >> shift);

drivers/iommu/intel/debugfs.c

543

invalidation_queue_entry_show(m, iommu);

drivers/iommu/intel/debugfs.c

555

struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

565

ri_entry = &iommu->ir_table->base[idx];

drivers/iommu/intel/debugfs.c

579

struct intel_iommu *iommu)

drivers/iommu/intel/debugfs.c

589

pi_entry = &iommu->ir_table->base[idx];

drivers/iommu/intel/debugfs.c

611

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

616

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

617

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/debugfs.c

621

iommu->name);

drivers/iommu/intel/debugfs.c

623

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/debugfs.c

624

if (iommu->ir_table && (sts & DMA_GSTS_IRES)) {

drivers/iommu/intel/debugfs.c

625

irta = virt_to_phys(iommu->ir_table->base);

drivers/iommu/intel/debugfs.c

627

ir_tbl_remap_entry_show(m, iommu);

drivers/iommu/intel/debugfs.c

636

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

637

if (!cap_pi_support(iommu->cap))

drivers/iommu/intel/debugfs.c

641

iommu->name);

drivers/iommu/intel/debugfs.c

643

if (iommu->ir_table) {

drivers/iommu/intel/debugfs.c

644

irta = virt_to_phys(iommu->ir_table->base);

drivers/iommu/intel/debugfs.c

646

ir_tbl_posted_entry_show(m, iommu);

drivers/iommu/intel/debugfs.c

659

static void latency_show_one(struct seq_file *m, struct intel_iommu *iommu,

drivers/iommu/intel/debugfs.c

663

iommu->name, drhd->reg_base_addr);

drivers/iommu/intel/debugfs.c

665

dmar_latency_snapshot(iommu, debug_buf, DEBUG_BUFFER_SIZE);

drivers/iommu/intel/debugfs.c

672

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

675

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

676

latency_show_one(m, iommu, drhd);

drivers/iommu/intel/debugfs.c

692

struct intel_iommu *iommu;

drivers/iommu/intel/debugfs.c

710

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/debugfs.c

711

dmar_latency_disable(iommu, DMAR_LATENCY_INV_IOTLB);

drivers/iommu/intel/debugfs.c

712

dmar_latency_disable(iommu, DMAR_LATENCY_INV_DEVTLB);

drivers/iommu/intel/debugfs.c

713

dmar_latency_disable(iommu, DMAR_LATENCY_INV_IEC);

drivers/iommu/intel/debugfs.c

719

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

720

dmar_latency_enable(iommu, DMAR_LATENCY_INV_IOTLB);

drivers/iommu/intel/debugfs.c

725

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

726

dmar_latency_enable(iommu, DMAR_LATENCY_INV_DEVTLB);

drivers/iommu/intel/debugfs.c

731

for_each_active_iommu(iommu, drhd)

drivers/iommu/intel/debugfs.c

732

dmar_latency_enable(iommu, DMAR_LATENCY_INV_IEC);

drivers/iommu/intel/dmar.c

1000

release_mem_region(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

1001

iommu->reg_size = map_size;

drivers/iommu/intel/dmar.c

1002

if (!request_mem_region(iommu->reg_phys, iommu->reg_size,

drivers/iommu/intel/dmar.c

1003

iommu->name)) {

drivers/iommu/intel/dmar.c

1008

iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

1009

if (!iommu->reg) {

drivers/iommu/intel/dmar.c

1016

if (cap_ecmds(iommu->cap)) {

drivers/iommu/intel/dmar.c

1020

iommu->ecmdcap[i] = readq(iommu->reg + DMAR_ECCAP_REG +

drivers/iommu/intel/dmar.c

1029

iounmap(iommu->reg);

drivers/iommu/intel/dmar.c

1031

release_mem_region(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

1038

struct intel_iommu *iommu;

drivers/iommu/intel/dmar.c

1049

iommu = kzalloc_obj(*iommu);

drivers/iommu/intel/dmar.c

1050

if (!iommu)

drivers/iommu/intel/dmar.c

1053

iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,

drivers/iommu/intel/dmar.c

1055

if (iommu->seq_id < 0) {

drivers/iommu/intel/dmar.c

1057

err = iommu->seq_id;

drivers/iommu/intel/dmar.c

1060

snprintf(iommu->name, sizeof(iommu->name), "dmar%d", iommu->seq_id);

drivers/iommu/intel/dmar.c

1062

err = map_iommu(iommu, drhd);

drivers/iommu/intel/dmar.c

1064

pr_err("Failed to map %s\n", iommu->name);

drivers/iommu/intel/dmar.c

1068

if (!cap_sagaw(iommu->cap) &&

drivers/iommu/intel/dmar.c

1069

(!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) {

drivers/iommu/intel/dmar.c

1071

iommu->name);

drivers/iommu/intel/dmar.c

1076

agaw = iommu_calculate_agaw(iommu);

drivers/iommu/intel/dmar.c

1079

iommu->seq_id);

drivers/iommu/intel/dmar.c

1084

msagaw = iommu_calculate_max_sagaw(iommu);

drivers/iommu/intel/dmar.c

1087

iommu->seq_id);

drivers/iommu/intel/dmar.c

1092

iommu->agaw = agaw;

drivers/iommu/intel/dmar.c

1093

iommu->msagaw = msagaw;

drivers/iommu/intel/dmar.c

1094

iommu->segment = drhd->segment;

drivers/iommu/intel/dmar.c

1095

iommu->device_rbtree = RB_ROOT;

drivers/iommu/intel/dmar.c

1096

spin_lock_init(&iommu->device_rbtree_lock);

drivers/iommu/intel/dmar.c

1097

mutex_init(&iommu->iopf_lock);

drivers/iommu/intel/dmar.c

1098

iommu->node = NUMA_NO_NODE;

drivers/iommu/intel/dmar.c

1099

spin_lock_init(&iommu->lock);

drivers/iommu/intel/dmar.c

1100

ida_init(&iommu->domain_ida);

drivers/iommu/intel/dmar.c

1101

mutex_init(&iommu->did_lock);

drivers/iommu/intel/dmar.c

1103

ver = readl(iommu->reg + DMAR_VER_REG);

drivers/iommu/intel/dmar.c

1105

iommu->name,

drivers/iommu/intel/dmar.c

1108

(unsigned long long)iommu->cap,

drivers/iommu/intel/dmar.c

1109

(unsigned long long)iommu->ecap);

drivers/iommu/intel/dmar.c

1112

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/dmar.c

1114

iommu->gcmd |= DMA_GCMD_IRE;

drivers/iommu/intel/dmar.c

1116

iommu->gcmd |= DMA_GCMD_TE;

drivers/iommu/intel/dmar.c

1118

iommu->gcmd |= DMA_GCMD_QIE;

drivers/iommu/intel/dmar.c

1120

if (alloc_iommu_pmu(iommu))

drivers/iommu/intel/dmar.c

1121

pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id);

drivers/iommu/intel/dmar.c

1123

raw_spin_lock_init(&iommu->register_lock);

drivers/iommu/intel/dmar.c

1129

if (pasid_supported(iommu))

drivers/iommu/intel/dmar.c

1130

iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);

drivers/iommu/intel/dmar.c

1138

err = iommu_device_sysfs_add(&iommu->iommu, NULL,

drivers/iommu/intel/dmar.c

1140

"%s", iommu->name);

drivers/iommu/intel/dmar.c

1144

err = iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);

drivers/iommu/intel/dmar.c

1148

iommu_pmu_register(iommu);

drivers/iommu/intel/dmar.c

1151

drhd->iommu = iommu;

drivers/iommu/intel/dmar.c

1152

iommu->drhd = drhd;

drivers/iommu/intel/dmar.c

1157

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/intel/dmar.c

1159

free_iommu_pmu(iommu);

drivers/iommu/intel/dmar.c

1160

unmap_iommu(iommu);

drivers/iommu/intel/dmar.c

1162

ida_free(&dmar_seq_ids, iommu->seq_id);

drivers/iommu/intel/dmar.c

1164

kfree(iommu);

drivers/iommu/intel/dmar.c

1168

static void free_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1170

if (intel_iommu_enabled && !iommu->drhd->ignored) {

drivers/iommu/intel/dmar.c

1171

iommu_pmu_unregister(iommu);

drivers/iommu/intel/dmar.c

1172

iommu_device_unregister(&iommu->iommu);

drivers/iommu/intel/dmar.c

1173

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/intel/dmar.c

1176

free_iommu_pmu(iommu);

drivers/iommu/intel/dmar.c

1178

if (iommu->irq) {

drivers/iommu/intel/dmar.c

1179

if (iommu->pr_irq) {

drivers/iommu/intel/dmar.c

1180

free_irq(iommu->pr_irq, iommu);

drivers/iommu/intel/dmar.c

1181

dmar_free_hwirq(iommu->pr_irq);

drivers/iommu/intel/dmar.c

1182

iommu->pr_irq = 0;

drivers/iommu/intel/dmar.c

1184

free_irq(iommu->irq, iommu);

drivers/iommu/intel/dmar.c

1185

dmar_free_hwirq(iommu->irq);

drivers/iommu/intel/dmar.c

1186

iommu->irq = 0;

drivers/iommu/intel/dmar.c

1189

if (iommu->qi) {

drivers/iommu/intel/dmar.c

1190

iommu_free_pages(iommu->qi->desc);

drivers/iommu/intel/dmar.c

1191

kfree(iommu->qi->desc_status);

drivers/iommu/intel/dmar.c

1192

kfree(iommu->qi);

drivers/iommu/intel/dmar.c

1195

if (iommu->reg)

drivers/iommu/intel/dmar.c

1196

unmap_iommu(iommu);

drivers/iommu/intel/dmar.c

1198

ida_destroy(&iommu->domain_ida);

drivers/iommu/intel/dmar.c

1199

ida_free(&dmar_seq_ids, iommu->seq_id);

drivers/iommu/intel/dmar.c

1200

kfree(iommu);

drivers/iommu/intel/dmar.c

1240

static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)

drivers/iommu/intel/dmar.c

1242

unsigned int head = readl(iommu->reg + DMAR_IQH_REG);

drivers/iommu/intel/dmar.c

1243

u64 iqe_err = readq(iommu->reg + DMAR_IQER_REG);

drivers/iommu/intel/dmar.c

1244

struct qi_desc *desc = iommu->qi->desc + head;

drivers/iommu/intel/dmar.c

1261

head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH;

drivers/iommu/intel/dmar.c

1262

head <<= qi_shift(iommu);

drivers/iommu/intel/dmar.c

1263

desc = iommu->qi->desc + head;

drivers/iommu/intel/dmar.c

1271

static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)

drivers/iommu/intel/dmar.c

1277

struct q_inval *qi = iommu->qi;

drivers/iommu/intel/dmar.c

1278

int shift = qi_shift(iommu);

drivers/iommu/intel/dmar.c

1283

fault = readl(iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1285

qi_dump_fault(iommu, fault);

drivers/iommu/intel/dmar.c

1293

head = readl(iommu->reg + DMAR_IQH_REG);

drivers/iommu/intel/dmar.c

1304

writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1315

head = readl(iommu->reg + DMAR_IQH_REG);

drivers/iommu/intel/dmar.c

1317

tail = readl(iommu->reg + DMAR_IQT_REG);

drivers/iommu/intel/dmar.c

1324

iqe_err = readq(iommu->reg + DMAR_IQER_REG);

drivers/iommu/intel/dmar.c

1327

writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1344

dev = device_rbtree_find(iommu, ite_sid);

drivers/iommu/intel/dmar.c

1354

writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1368

int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,

drivers/iommu/intel/dmar.c

1371

struct q_inval *qi = iommu->qi;

drivers/iommu/intel/dmar.c

1388

dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IOTLB))

drivers/iommu/intel/dmar.c

1392

dmar_latency_enabled(iommu, DMAR_LATENCY_INV_DEVTLB))

drivers/iommu/intel/dmar.c

1396

dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IEC))

drivers/iommu/intel/dmar.c

1416

shift = qi_shift(iommu);

drivers/iommu/intel/dmar.c

1422

trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1,

drivers/iommu/intel/dmar.c

1445

writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);

drivers/iommu/intel/dmar.c

1455

rc = qi_check_fault(iommu, index, wait_index);

drivers/iommu/intel/dmar.c

1482

dmar_latency_update(iommu, DMAR_LATENCY_INV_IOTLB,

drivers/iommu/intel/dmar.c

1486

dmar_latency_update(iommu, DMAR_LATENCY_INV_DEVTLB,

drivers/iommu/intel/dmar.c

1490

dmar_latency_update(iommu, DMAR_LATENCY_INV_IEC,

drivers/iommu/intel/dmar.c

1499

void qi_global_iec(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1509

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1512

void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,

drivers/iommu/intel/dmar.c

1523

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1526

void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/dmar.c

1531

qi_desc_iotlb(iommu, did, addr, size_order, type, &desc);

drivers/iommu/intel/dmar.c

1532

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1535

void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/dmar.c

1546

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/dmar.c

1550

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1554

void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid)

drivers/iommu/intel/dmar.c

1559

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1563

void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/dmar.c

1574

if (!(iommu->gcmd & DMA_GCMD_TE))

drivers/iommu/intel/dmar.c

1580

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1583

void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,

drivers/iommu/intel/dmar.c

1590

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/dmar.c

1596

void dmar_disable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1602

if (!ecap_qis(iommu->ecap))

drivers/iommu/intel/dmar.c

1605

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1607

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/dmar.c

1614

while ((readl(iommu->reg + DMAR_IQT_REG) !=

drivers/iommu/intel/dmar.c

1615

readl(iommu->reg + DMAR_IQH_REG)) &&

drivers/iommu/intel/dmar.c

1619

iommu->gcmd &= ~DMA_GCMD_QIE;

drivers/iommu/intel/dmar.c

1620

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/dmar.c

1622

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,

drivers/iommu/intel/dmar.c

1625

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1631

static void __dmar_enable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1635

struct q_inval *qi = iommu->qi;

drivers/iommu/intel/dmar.c

1645

if (ecap_smts(iommu->ecap))

drivers/iommu/intel/dmar.c

1648

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1651

writel(0, iommu->reg + DMAR_IQT_REG);

drivers/iommu/intel/dmar.c

1653

writeq(val, iommu->reg + DMAR_IQA_REG);

drivers/iommu/intel/dmar.c

1655

iommu->gcmd |= DMA_GCMD_QIE;

drivers/iommu/intel/dmar.c

1656

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/dmar.c

1659

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);

drivers/iommu/intel/dmar.c

1661

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/dmar.c

1669

int dmar_enable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

1674

if (!ecap_qis(iommu->ecap))

drivers/iommu/intel/dmar.c

1680

if (iommu->qi)

drivers/iommu/intel/dmar.c

1683

iommu->qi = kmalloc_obj(*qi, GFP_ATOMIC);

drivers/iommu/intel/dmar.c

1684

if (!iommu->qi)

drivers/iommu/intel/dmar.c

1687

qi = iommu->qi;

drivers/iommu/intel/dmar.c

1693

desc = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC,

drivers/iommu/intel/dmar.c

1694

ecap_smts(iommu->ecap) ? SZ_8K :

drivers/iommu/intel/dmar.c

1698

iommu->qi = NULL;

drivers/iommu/intel/dmar.c

1708

iommu->qi = NULL;

drivers/iommu/intel/dmar.c

1714

__dmar_enable_qi(iommu);

drivers/iommu/intel/dmar.c

1834

static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)

drivers/iommu/intel/dmar.c

1836

if (iommu->irq == irq)

drivers/iommu/intel/dmar.c

1838

else if (iommu->pr_irq == irq)

drivers/iommu/intel/dmar.c

1840

else if (iommu->perf_irq == irq)

drivers/iommu/intel/dmar.c

1848

struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);

drivers/iommu/intel/dmar.c

1849

int reg = dmar_msi_reg(iommu, data->irq);

drivers/iommu/intel/dmar.c

1853

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1854

writel(0, iommu->reg + reg);

drivers/iommu/intel/dmar.c

1856

readl(iommu->reg + reg);

drivers/iommu/intel/dmar.c

1857

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1862

struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);

drivers/iommu/intel/dmar.c

1863

int reg = dmar_msi_reg(iommu, data->irq);

drivers/iommu/intel/dmar.c

1867

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1868

writel(DMA_FECTL_IM, iommu->reg + reg);

drivers/iommu/intel/dmar.c

1870

readl(iommu->reg + reg);

drivers/iommu/intel/dmar.c

1871

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1876

struct intel_iommu *iommu = irq_get_handler_data(irq);

drivers/iommu/intel/dmar.c

1877

int reg = dmar_msi_reg(iommu, irq);

drivers/iommu/intel/dmar.c

1880

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1881

writel(msg->data, iommu->reg + reg + 4);

drivers/iommu/intel/dmar.c

1882

writel(msg->address_lo, iommu->reg + reg + 8);

drivers/iommu/intel/dmar.c

1883

writel(msg->address_hi, iommu->reg + reg + 12);

drivers/iommu/intel/dmar.c

1884

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1887

static int dmar_fault_do_one(struct intel_iommu *iommu, int type,

drivers/iommu/intel/dmar.c

1898

iommu->segment,

drivers/iommu/intel/dmar.c

1909

iommu->segment,

drivers/iommu/intel/dmar.c

1916

iommu->segment,

drivers/iommu/intel/dmar.c

1921

dmar_fault_dump_ptes(iommu, source_id, addr, pasid);

drivers/iommu/intel/dmar.c

1929

struct intel_iommu *iommu = dev_id;

drivers/iommu/intel/dmar.c

1937

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1938

fault_status = readl(iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

1947

reg = cap_fault_reg_offset(iommu->cap);

drivers/iommu/intel/dmar.c

1960

data = readl(iommu->reg + reg +

drivers/iommu/intel/dmar.c

1970

data = readl(iommu->reg + reg +

drivers/iommu/intel/dmar.c

1975

guest_addr = readq(iommu->reg + reg +

drivers/iommu/intel/dmar.c

1981

writel(DMA_FRCD_F, iommu->reg + reg +

drivers/iommu/intel/dmar.c

1984

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1988

dmar_fault_do_one(iommu, type, fault_reason,

drivers/iommu/intel/dmar.c

1993

if (fault_index >= cap_num_fault_regs(iommu->cap))

drivers/iommu/intel/dmar.c

1995

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

1999

iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

2002

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/dmar.c

2006

int dmar_set_interrupt(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

2013

if (iommu->irq)

drivers/iommu/intel/dmar.c

2016

irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);

drivers/iommu/intel/dmar.c

2018

iommu->irq = irq;

drivers/iommu/intel/dmar.c

2024

ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);

drivers/iommu/intel/dmar.c

2033

struct intel_iommu *iommu;

drivers/iommu/intel/dmar.c

2039

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/dmar.c

2043

if (iommu->irq || iommu->node != cpu_to_node(cpu))

drivers/iommu/intel/dmar.c

2046

ret = dmar_set_interrupt(iommu);

drivers/iommu/intel/dmar.c

2057

dmar_fault(iommu->irq, iommu);

drivers/iommu/intel/dmar.c

2058

fault_status = readl(iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

2059

writel(fault_status, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/intel/dmar.c

2068

int dmar_reenable_qi(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

2070

if (!ecap_qis(iommu->ecap))

drivers/iommu/intel/dmar.c

2073

if (!iommu->qi)

drivers/iommu/intel/dmar.c

2079

dmar_disable_qi(iommu);

drivers/iommu/intel/dmar.c

2085

__dmar_enable_qi(iommu);

drivers/iommu/intel/dmar.c

462

if (dmaru->iommu)

drivers/iommu/intel/dmar.c

463

free_iommu(dmaru->iommu);

drivers/iommu/intel/dmar.c

502

drhd->iommu->node = node;

drivers/iommu/intel/dmar.c

68

static void free_iommu(struct intel_iommu *iommu);

drivers/iommu/intel/dmar.c

939

x86_init.iommu.iommu_init = intel_iommu_init;

drivers/iommu/intel/dmar.c

950

static void unmap_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/dmar.c

952

iounmap(iommu->reg);

drivers/iommu/intel/dmar.c

953

release_mem_region(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

964

static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd)

drivers/iommu/intel/dmar.c

969

iommu->reg_phys = phys_addr;

drivers/iommu/intel/dmar.c

970

iommu->reg_size = drhd->reg_size;

drivers/iommu/intel/dmar.c

972

if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {

drivers/iommu/intel/dmar.c

978

iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);

drivers/iommu/intel/dmar.c

979

if (!iommu->reg) {

drivers/iommu/intel/dmar.c

985

iommu->cap = readq(iommu->reg + DMAR_CAP_REG);

drivers/iommu/intel/dmar.c

986

iommu->ecap = readq(iommu->reg + DMAR_ECAP_REG);

drivers/iommu/intel/dmar.c

988

if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {

drivers/iommu/intel/dmar.c

995

map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),

drivers/iommu/intel/dmar.c

996

cap_max_fault_reg_offset(iommu->cap));

drivers/iommu/intel/dmar.c

998

if (map_size > iommu->reg_size) {

drivers/iommu/intel/dmar.c

999

iounmap(iommu->reg);

drivers/iommu/intel/iommu.c

1000

bitmap_free(iommu->copied_tables);

drivers/iommu/intel/iommu.c

1001

iommu->copied_tables = NULL;

drivers/iommu/intel/iommu.c

1005

free_context_table(iommu);

drivers/iommu/intel/iommu.c

1007

if (ecap_prs(iommu->ecap))

drivers/iommu/intel/iommu.c

1008

intel_iommu_finish_prq(iommu);

drivers/iommu/intel/iommu.c

1015

static bool first_level_by_default(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1018

if (!sm_supported(iommu))

drivers/iommu/intel/iommu.c

1022

if (ecap_flts(iommu->ecap) ^ ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

1023

return ecap_flts(iommu->ecap);

drivers/iommu/intel/iommu.c

1028

int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1040

guard(mutex)(&iommu->did_lock);

drivers/iommu/intel/iommu.c

1041

curr = xa_load(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.c

1048

num = ida_alloc_range(&iommu->domain_ida, IDA_START_DID,

drivers/iommu/intel/iommu.c

1049

cap_ndoms(iommu->cap) - 1, GFP_KERNEL);

drivers/iommu/intel/iommu.c

1051

pr_err("%s: No free domain ids\n", iommu->name);

drivers/iommu/intel/iommu.c

1057

info->iommu = iommu;

drivers/iommu/intel/iommu.c

1058

curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,

drivers/iommu/intel/iommu.c

1068

ida_free(&iommu->domain_ida, info->did);

drivers/iommu/intel/iommu.c

1074

void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1081

guard(mutex)(&iommu->did_lock);

drivers/iommu/intel/iommu.c

1082

info = xa_load(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.c

1084

ida_free(&iommu->domain_ida, info->did);

drivers/iommu/intel/iommu.c

1085

xa_erase(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.c

1099

static void copied_context_tear_down(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1105

if (!context_copied(iommu, bus, devfn))

drivers/iommu/intel/iommu.c

1108

assert_spin_locked(&iommu->lock);

drivers/iommu/intel/iommu.c

1113

if (did_old < cap_ndoms(iommu->cap)) {

drivers/iommu/intel/iommu.c

1114

iommu->flush.flush_context(iommu, did_old,

drivers/iommu/intel/iommu.c

1118

iommu->flush.flush_iotlb(iommu, did_old, 0, 0,

drivers/iommu/intel/iommu.c

1122

clear_context_copied(iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1131

static void context_present_cache_flush(struct intel_iommu *iommu, u16 did,

drivers/iommu/intel/iommu.c

1134

if (cap_caching_mode(iommu->cap)) {

drivers/iommu/intel/iommu.c

1135

iommu->flush.flush_context(iommu, 0,

drivers/iommu/intel/iommu.c

1139

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/iommu.c

1141

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1146

struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1150

domain_lookup_dev_info(domain, iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1151

u16 did = domain_id_iommu(domain, iommu);

drivers/iommu/intel/iommu.c

1165

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

1167

context = iommu_context_addr(iommu, bus, devfn, 1);

drivers/iommu/intel/iommu.c

1172

if (context_present(context) && !context_copied(iommu, bus, devfn))

drivers/iommu/intel/iommu.c

1175

copied_context_tear_down(iommu, context, bus, devfn);

drivers/iommu/intel/iommu.c

1189

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/iommu.c

1191

context_present_cache_flush(iommu, did, bus, devfn);

drivers/iommu/intel/iommu.c

1195

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

1204

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1207

return domain_context_mapping_one(domain, iommu,

drivers/iommu/intel/iommu.c

1215

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1220

return domain_context_mapping_one(domain, iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1234

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1238

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

1239

context = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/iommu.c

124

struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid)

drivers/iommu/intel/iommu.c

1241

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

1247

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/iommu.c

1248

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

1251

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/iommu.c

1254

int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/iommu.c

1259

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

1261

return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, did, flags);

drivers/iommu/intel/iommu.c

1264

static int domain_setup_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1270

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

1272

return intel_pasid_setup_second_level(iommu, domain, dev, pasid);

drivers/iommu/intel/iommu.c

1275

static int domain_setup_passthrough(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1280

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

1282

return intel_pasid_setup_pass_through(iommu, dev, pasid);

drivers/iommu/intel/iommu.c

1285

static int domain_setup_first_level(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

130

spin_lock_irqsave(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1307

return __domain_setup_first_level(iommu, dev, pasid,

drivers/iommu/intel/iommu.c

1308

domain_id_iommu(domain, iommu),

drivers/iommu/intel/iommu.c

131

node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key);

drivers/iommu/intel/iommu.c

1316

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1320

ret = domain_attach_iommu(domain, iommu);

drivers/iommu/intel/iommu.c

1333

if (!sm_supported(iommu))

drivers/iommu/intel/iommu.c

1336

ret = domain_setup_first_level(iommu, domain, dev,

drivers/iommu/intel/iommu.c

1339

ret = domain_setup_second_level(iommu, domain, dev,

drivers/iommu/intel/iommu.c

134

spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

139

static int device_rbtree_insert(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

1390

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1396

if (!ecap_pass_through(iommu->ecap))

drivers/iommu/intel/iommu.c

1409

static void intel_iommu_init_qi(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1417

if (!iommu->qi) {

drivers/iommu/intel/iommu.c

1421

dmar_fault(-1, iommu);

drivers/iommu/intel/iommu.c

1426

dmar_disable_qi(iommu);

drivers/iommu/intel/iommu.c

1429

if (dmar_enable_qi(iommu)) {

drivers/iommu/intel/iommu.c

1433

iommu->flush.flush_context = __iommu_flush_context;

drivers/iommu/intel/iommu.c

1434

iommu->flush.flush_iotlb = __iommu_flush_iotlb;

drivers/iommu/intel/iommu.c

1436

iommu->name);

drivers/iommu/intel/iommu.c

1438

iommu->flush.flush_context = qi_flush_context;

drivers/iommu/intel/iommu.c

1439

iommu->flush.flush_iotlb = qi_flush_iotlb;

drivers/iommu/intel/iommu.c

1440

pr_info("%s: Using Queued invalidation\n", iommu->name);

drivers/iommu/intel/iommu.c

1444

static int copy_context_table(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

145

spin_lock_irqsave(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

146

curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp);

drivers/iommu/intel/iommu.c

1466

__iommu_flush_cache(iommu, new_ce,

drivers/iommu/intel/iommu.c

147

spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1496

new_ce = iommu_alloc_pages_node_sz(iommu->node,

drivers/iommu/intel/iommu.c

1511

if (did >= 0 && did < cap_ndoms(iommu->cap))

drivers/iommu/intel/iommu.c

1512

ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);

drivers/iommu/intel/iommu.c

1514

set_context_copied(iommu, bus, devfn);

drivers/iommu/intel/iommu.c

1520

__iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);

drivers/iommu/intel/iommu.c

1529

static int copy_translation_tables(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

1539

rtaddr_reg = readq(iommu->reg + DMAR_RTADDR_REG);

drivers/iommu/intel/iommu.c

1541

new_ext = !!sm_supported(iommu);

drivers/iommu/intel/iommu.c

1552

iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);

drivers/iommu/intel/iommu.c

1553

if (!iommu->copied_tables)

drivers/iommu/intel/iommu.c

156

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

1572

ret = copy_context_table(iommu, &old_rt[bus],

drivers/iommu/intel/iommu.c

1576

iommu->name, bus);

drivers/iommu/intel/iommu.c

1581

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

159

spin_lock_irqsave(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1590

iommu->root_entry[bus].lo = val;

drivers/iommu/intel/iommu.c

1597

iommu->root_entry[bus].hi = val;

drivers/iommu/intel/iommu.c

1600

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

1604

__iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);

drivers/iommu/intel/iommu.c

161

rb_erase(&info->node, &iommu->device_rbtree);

drivers/iommu/intel/iommu.c

1617

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

1620

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1622

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1631

if (pasid_supported(iommu)) {

drivers/iommu/intel/iommu.c

1632

u32 temp = 2 << ecap_pss(iommu->ecap);

drivers/iommu/intel/iommu.c

1638

intel_iommu_init_qi(iommu);

drivers/iommu/intel/iommu.c

1639

init_translation_status(iommu);

drivers/iommu/intel/iommu.c

164

spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);

drivers/iommu/intel/iommu.c

1641

if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {

drivers/iommu/intel/iommu.c

1642

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1643

clear_translation_pre_enabled(iommu);

drivers/iommu/intel/iommu.c

1645

iommu->name);

drivers/iommu/intel/iommu.c

1653

ret = iommu_alloc_root_entry(iommu);

drivers/iommu/intel/iommu.c

1657

if (translation_pre_enabled(iommu)) {

drivers/iommu/intel/iommu.c

1660

ret = copy_translation_tables(iommu);

drivers/iommu/intel/iommu.c

1672

iommu->name);

drivers/iommu/intel/iommu.c

1673

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1674

clear_translation_pre_enabled(iommu);

drivers/iommu/intel/iommu.c

1677

iommu->name);

drivers/iommu/intel/iommu.c

1681

intel_svm_check(iommu);

drivers/iommu/intel/iommu.c

1689

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1690

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1691

iommu_set_root_entry(iommu);

drivers/iommu/intel/iommu.c

1703

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1710

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

1714

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1716

if (ecap_prs(iommu->ecap)) {

drivers/iommu/intel/iommu.c

1722

ret = intel_iommu_enable_prq(iommu);

drivers/iommu/intel/iommu.c

1728

ret = dmar_set_interrupt(iommu);

drivers/iommu/intel/iommu.c

1736

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1737

disable_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

1738

free_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

1784

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

1787

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1788

if (iommu->qi) {

drivers/iommu/intel/iommu.c

1789

ret = dmar_reenable_qi(iommu);

drivers/iommu/intel/iommu.c

1795

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1802

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

1806

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

1807

iommu_set_root_entry(iommu);

drivers/iommu/intel/iommu.c

1808

iommu_enable_translation(iommu);

drivers/iommu/intel/iommu.c

1809

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

1818

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

1820

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1821

iommu->flush.flush_context(iommu, 0, 0, 0,

drivers/iommu/intel/iommu.c

1823

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

drivers/iommu/intel/iommu.c

1831

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

1836

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1837

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

1839

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

1841

iommu->iommu_state[SR_DMAR_FECTL_REG] =

drivers/iommu/intel/iommu.c

1842

readl(iommu->reg + DMAR_FECTL_REG);

drivers/iommu/intel/iommu.c

1843

iommu->iommu_state[SR_DMAR_FEDATA_REG] =

drivers/iommu/intel/iommu.c

1844

readl(iommu->reg + DMAR_FEDATA_REG);

drivers/iommu/intel/iommu.c

1845

iommu->iommu_state[SR_DMAR_FEADDR_REG] =

drivers/iommu/intel/iommu.c

1846

readl(iommu->reg + DMAR_FEADDR_REG);

drivers/iommu/intel/iommu.c

1847

iommu->iommu_state[SR_DMAR_FEUADDR_REG] =

drivers/iommu/intel/iommu.c

1848

readl(iommu->reg + DMAR_FEUADDR_REG);

drivers/iommu/intel/iommu.c

1850

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

1858

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

1869

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

1871

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

1873

writel(iommu->iommu_state[SR_DMAR_FECTL_REG],

drivers/iommu/intel/iommu.c

1874

iommu->reg + DMAR_FECTL_REG);

drivers/iommu/intel/iommu.c

1875

writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],

drivers/iommu/intel/iommu.c

1876

iommu->reg + DMAR_FEDATA_REG);

drivers/iommu/intel/iommu.c

1877

writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],

drivers/iommu/intel/iommu.c

1878

iommu->reg + DMAR_FEADDR_REG);

drivers/iommu/intel/iommu.c

1879

writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],

drivers/iommu/intel/iommu.c

188

struct intel_iommu *iommu; /* the corresponding iommu */

drivers/iommu/intel/iommu.c

1880

iommu->reg + DMAR_FEUADDR_REG);

drivers/iommu/intel/iommu.c

1882

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

2111

struct intel_iommu *iommu = dmaru->iommu;

drivers/iommu/intel/iommu.c

2117

if (iommu->gcmd & DMA_GCMD_TE)

drivers/iommu/intel/iommu.c

2118

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

2120

ret = iommu_alloc_root_entry(iommu);

drivers/iommu/intel/iommu.c

2124

intel_svm_check(iommu);

drivers/iommu/intel/iommu.c

2131

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2135

intel_iommu_init_qi(iommu);

drivers/iommu/intel/iommu.c

2136

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/iommu.c

2138

if (ecap_prs(iommu->ecap)) {

drivers/iommu/intel/iommu.c

2139

ret = intel_iommu_enable_prq(iommu);

drivers/iommu/intel/iommu.c

2144

ret = dmar_set_interrupt(iommu);

drivers/iommu/intel/iommu.c

2148

iommu_set_root_entry(iommu);

drivers/iommu/intel/iommu.c

2149

iommu_enable_translation(iommu);

drivers/iommu/intel/iommu.c

2151

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2155

disable_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

2157

free_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

2164

struct intel_iommu *iommu = dmaru->iommu;

drivers/iommu/intel/iommu.c

2168

if (iommu == NULL)

drivers/iommu/intel/iommu.c

217

static bool translation_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

2174

disable_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

2175

free_dmar_iommu(iommu);

drivers/iommu/intel/iommu.c

219

return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);

drivers/iommu/intel/iommu.c

222

static void clear_translation_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

2227

static bool dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

224

iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/intel/iommu.c

2248

return !(satcu->atc_required && !sm_supported(iommu));

drivers/iommu/intel/iommu.c

227

static void init_translation_status(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

231

gsts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/iommu.c

233

iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;

drivers/iommu/intel/iommu.c

2356

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

2359

for_each_iommu(iommu, drhd)

drivers/iommu/intel/iommu.c

2360

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

2366

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/iommu.c

2376

iommu = drhd->iommu;

drivers/iommu/intel/iommu.c

2379

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2382

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

2390

return container_of(iommu_dev, struct intel_iommu, iommu);

drivers/iommu/intel/iommu.c

2396

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2397

u32 ver = readl(iommu->reg + DMAR_VER_REG);

drivers/iommu/intel/iommu.c

2406

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2407

return sysfs_emit(buf, "%llx\n", iommu->reg_phys);

drivers/iommu/intel/iommu.c

2414

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2415

return sysfs_emit(buf, "%llx\n", iommu->cap);

drivers/iommu/intel/iommu.c

2422

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2423

return sysfs_emit(buf, "%llx\n", iommu->ecap);

drivers/iommu/intel/iommu.c

2430

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2431

return sysfs_emit(buf, "%ld\n", cap_ndoms(iommu->cap));

drivers/iommu/intel/iommu.c

2438

struct intel_iommu *iommu = dev_to_intel_iommu(dev);

drivers/iommu/intel/iommu.c

2442

for (id = 0; id < cap_ndoms(iommu->cap); id++)

drivers/iommu/intel/iommu.c

2443

if (ida_exists(&iommu->domain_ida, id))

drivers/iommu/intel/iommu.c

2508

struct intel_iommu *iommu __maybe_unused;

drivers/iommu/intel/iommu.c

2512

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

2559

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

2604

for_each_iommu(iommu, drhd)

drivers/iommu/intel/iommu.c

2605

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2640

for_each_active_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

2648

if (cap_caching_mode(iommu->cap) &&

drivers/iommu/intel/iommu.c

2649

!first_level_by_default(iommu)) {

drivers/iommu/intel/iommu.c

2653

iommu_device_sysfs_add(&iommu->iommu, NULL,

drivers/iommu/intel/iommu.c

2655

"%s", iommu->name);

drivers/iommu/intel/iommu.c

2662

iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);

drivers/iommu/intel/iommu.c

2665

iommu_pmu_register(iommu);

drivers/iommu/intel/iommu.c

2672

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

2673

if (!drhd->ignored && !translation_pre_enabled(iommu))

drivers/iommu/intel/iommu.c

2674

iommu_enable_translation(iommu);

drivers/iommu/intel/iommu.c

2676

iommu_disable_protect_mem_regions(iommu);

drivers/iommu/intel/iommu.c

2726

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

2737

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

2738

intel_pasid_tear_down_entry(iommu, dev,

drivers/iommu/intel/iommu.c

2754

domain_detach_iommu(info->domain, iommu);

drivers/iommu/intel/iommu.c

2801

static unsigned int compute_vasz_lg2_fs(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

2804

unsigned int mgaw = cap_mgaw(iommu->cap);

drivers/iommu/intel/iommu.c

2813

if (mgaw > 48 && cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/iommu.c

2825

struct intel_iommu *iommu, u32 flags)

drivers/iommu/intel/iommu.c

2835

if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))

drivers/iommu/intel/iommu.c

2843

compute_vasz_lg2_fs(iommu, &cfg.top_level);

drivers/iommu/intel/iommu.c

2848

if (!ecap_smpwc(iommu->ecap))

drivers/iommu/intel/iommu.c

2850

dmar_domain->iommu.iommu_device = dev;

drivers/iommu/intel/iommu.c

2851

dmar_domain->iommu.nid = dev_to_node(dev);

drivers/iommu/intel/iommu.c

2858

if (rwbf_required(iommu))

drivers/iommu/intel/iommu.c

2867

if (!cap_fl1gp_support(iommu->cap))

drivers/iommu/intel/iommu.c

2875

static unsigned int compute_vasz_lg2_ss(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

2878

unsigned int sagaw = cap_sagaw(iommu->cap);

drivers/iommu/intel/iommu.c

2879

unsigned int mgaw = cap_mgaw(iommu->cap);

drivers/iommu/intel/iommu.c

288

static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

2907

struct intel_iommu *iommu, u32 flags)

drivers/iommu/intel/iommu.c

292

fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);

drivers/iommu/intel/iommu.c

2920

!nested_supported(iommu)) ||

drivers/iommu/intel/iommu.c

2922

!ssads_supported(iommu)))

drivers/iommu/intel/iommu.c

2926

if (sm_supported(iommu) && !ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

293

sl_sagaw = cap_sagaw(iommu->cap);

drivers/iommu/intel/iommu.c

2933

cfg.common.hw_max_vasz_lg2 = compute_vasz_lg2_ss(iommu, &cfg.top_level);

drivers/iommu/intel/iommu.c

2945

if (!iommu_paging_structure_coherency(iommu))

drivers/iommu/intel/iommu.c

2947

dmar_domain->iommu.iommu_device = dev;

drivers/iommu/intel/iommu.c

2948

dmar_domain->iommu.nid = dev_to_node(dev);

drivers/iommu/intel/iommu.c

296

if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))

drivers/iommu/intel/iommu.c

2962

sslps = cap_super_page_val(iommu->cap);

drivers/iommu/intel/iommu.c

2975

if (rwbf_required(iommu) || cap_caching_mode(iommu->cap))

drivers/iommu/intel/iommu.c

2986

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

2993

domain = intel_iommu_domain_alloc_first_stage(dev, iommu, flags);

drivers/iommu/intel/iommu.c

2996

return intel_iommu_domain_alloc_second_stage(dev, iommu, flags);

drivers/iommu/intel/iommu.c

300

if (!ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

3010

pt_iommu_deinit(&dmar_domain->iommu);

drivers/iommu/intel/iommu.c

3017

struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3024

if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))

drivers/iommu/intel/iommu.c

3027

if (!ecap_smpwc(iommu->ecap) &&

drivers/iommu/intel/iommu.c

3033

if (!cap_fl5lp_support(iommu->cap) &&

drivers/iommu/intel/iommu.c

3038

if (!cap_fl1gp_support(iommu->cap) &&

drivers/iommu/intel/iommu.c

3043

if ((rwbf_required(iommu)) && !dmar_domain->iotlb_sync_map)

drivers/iommu/intel/iommu.c

3051

struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3054

unsigned int sslps = cap_super_page_val(iommu->cap);

drivers/iommu/intel/iommu.c

3059

if (dmar_domain->domain.dirty_ops && !ssads_supported(iommu))

drivers/iommu/intel/iommu.c

306

static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)

drivers/iommu/intel/iommu.c

3061

if (dmar_domain->nested_parent && !nested_supported(iommu))

drivers/iommu/intel/iommu.c

3065

if (sm_supported(iommu) && !ecap_slts(iommu->ecap))

drivers/iommu/intel/iommu.c

3068

if (!iommu_paging_structure_coherency(iommu) &&

drivers/iommu/intel/iommu.c

3074

if (cap_mgaw(iommu->cap) < vasz_lg2)

drivers/iommu/intel/iommu.c

3078

if (!(cap_sagaw(iommu->cap) & BIT(pt_info.aw)))

drivers/iommu/intel/iommu.c

3088

if ((rwbf_required(iommu) || cap_caching_mode(iommu->cap)) &&

drivers/iommu/intel/iommu.c

3098

!ecap_sc_support(iommu->ecap))

drivers/iommu/intel/iommu.c

3107

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

311

sagaw = __iommu_calculate_sagaw(iommu);

drivers/iommu/intel/iommu.c

3111

ret = paging_domain_compatible_first_stage(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3113

ret = paging_domain_compatible_second_stage(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3119

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&

drivers/iommu/intel/iommu.c

3120

context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/iommu.c

3165

if (!ecap_sc_support(info->iommu->ecap)) {

drivers/iommu/intel/iommu.c

3189

intel_pasid_setup_page_snoop_control(info->iommu, info->dev,

drivers/iommu/intel/iommu.c

3222

return ecap_sc_support(info->iommu->ecap);

drivers/iommu/intel/iommu.c

3224

return ssads_supported(info->iommu);

drivers/iommu/intel/iommu.c

323

int iommu_calculate_max_sagaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3236

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

3240

iommu = device_lookup_iommu(dev, &bus, &devfn);

drivers/iommu/intel/iommu.c

3241

if (!iommu || !iommu->iommu.ops)

drivers/iommu/intel/iommu.c

325

return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);

drivers/iommu/intel/iommu.c

3255

info->segment = iommu->segment;

drivers/iommu/intel/iommu.c

3259

info->iommu = iommu;

drivers/iommu/intel/iommu.c

3262

if (ecap_dev_iotlb_support(iommu->ecap) &&

drivers/iommu/intel/iommu.c

3264

dmar_ats_supported(pdev, iommu)) {

drivers/iommu/intel/iommu.c

3275

if (ecap_dit(iommu->ecap))

drivers/iommu/intel/iommu.c

3279

if (sm_supported(iommu)) {

drivers/iommu/intel/iommu.c

3280

if (pasid_supported(iommu)) {

drivers/iommu/intel/iommu.c

3287

if (info->ats_supported && ecap_prs(iommu->ecap) &&

drivers/iommu/intel/iommu.c

3288

ecap_pds(iommu->ecap) && pci_pri_supported(pdev))

drivers/iommu/intel/iommu.c

3296

ret = device_rbtree_insert(iommu, info);

drivers/iommu/intel/iommu.c

3301

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {

drivers/iommu/intel/iommu.c

3308

if (!context_copied(iommu, info->bus, info->devfn)) {

drivers/iommu/intel/iommu.c

3317

return &iommu->iommu;

drivers/iommu/intel/iommu.c

333

int iommu_calculate_agaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

3331

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3343

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {

drivers/iommu/intel/iommu.c

3347

u16 did = domain_id_iommu(info->domain, iommu);

drivers/iommu/intel/iommu.c

335

return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);

drivers/iommu/intel/iommu.c

3360

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3370

mutex_lock(&iommu->iopf_lock);

drivers/iommu/intel/iommu.c

3373

mutex_unlock(&iommu->iopf_lock);

drivers/iommu/intel/iommu.c

3375

if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&

drivers/iommu/intel/iommu.c

3376

!context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/iommu.c

338

static bool iommu_paging_structure_coherency(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

340

return sm_supported(iommu) ?

drivers/iommu/intel/iommu.c

341

ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);

drivers/iommu/intel/iommu.c

344

struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,

drivers/iommu/intel/iommu.c

3453

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3466

ret = iopf_queue_add_device(iommu->iopf_queue, dev);

drivers/iommu/intel/iommu.c

347

struct root_entry *root = &iommu->root_entry[bus];

drivers/iommu/intel/iommu.c

3478

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3487

iopf_queue_remove_device(iommu->iopf_queue, dev);

drivers/iommu/intel/iommu.c

3494

return translation_pre_enabled(info->iommu) && !info->domain;

drivers/iommu/intel/iommu.c

3530

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

355

if (!alloc && context_copied(iommu, bus, devfn))

drivers/iommu/intel/iommu.c

3556

domain_detach_iommu(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3567

intel_pasid_tear_down_entry(info->iommu, dev, pasid, false);

drivers/iommu/intel/iommu.c

3580

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3589

ret = domain_attach_iommu(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

359

if (sm_supported(iommu)) {

drivers/iommu/intel/iommu.c

3605

domain_detach_iommu(dmar_domain, iommu);

drivers/iommu/intel/iommu.c

3617

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3624

if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))

drivers/iommu/intel/iommu.c

3627

if (context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/iommu.c

3643

ret = domain_setup_first_level(iommu, dmar_domain,

drivers/iommu/intel/iommu.c

3646

ret = domain_setup_second_level(iommu, dmar_domain,

drivers/iommu/intel/iommu.c

3671

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3683

vtd->cap_reg = iommu->cap;

drivers/iommu/intel/iommu.c

3684

vtd->ecap_reg = iommu->ecap;

drivers/iommu/intel/iommu.c

3700

ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev,

drivers/iommu/intel/iommu.c

3708

ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev,

drivers/iommu/intel/iommu.c

373

context = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC,

drivers/iommu/intel/iommu.c

378

__iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);

drivers/iommu/intel/iommu.c

3780

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3783

spin_lock(&iommu->lock);

drivers/iommu/intel/iommu.c

3784

context = iommu_context_addr(iommu, bus, devfn, 1);

drivers/iommu/intel/iommu.c

3786

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

3790

if (context_present(context) && !context_copied(iommu, bus, devfn)) {

drivers/iommu/intel/iommu.c

3791

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

3795

copied_context_tear_down(iommu, context, bus, devfn);

drivers/iommu/intel/iommu.c

3803

context_set_address_width(context, iommu->msagaw);

drivers/iommu/intel/iommu.c

3807

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/iommu.c

3809

context_present_cache_flush(iommu, FLPT_DEFAULT_DID, bus, devfn);

drivers/iommu/intel/iommu.c

381

__iommu_flush_cache(iommu, entry, sizeof(*entry));

drivers/iommu/intel/iommu.c

3810

spin_unlock(&iommu->lock);

drivers/iommu/intel/iommu.c

3838

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3851

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

3852

ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);

drivers/iommu/intel/iommu.c

3867

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/iommu.c

3870

if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))

drivers/iommu/intel/iommu.c

3877

ret = domain_setup_passthrough(iommu, dev, pasid, old);

drivers/iommu/intel/iommu.c

4167

qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,

drivers/iommu/intel/iommu.c

4170

qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,

drivers/iommu/intel/iommu.c

4189

int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob)

drivers/iommu/intel/iommu.c

4195

if (!cap_ecmds(iommu->cap))

drivers/iommu/intel/iommu.c

4198

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

4200

res = readq(iommu->reg + DMAR_ECRSP_REG);

drivers/iommu/intel/iommu.c

4213

writeq(ob, iommu->reg + DMAR_ECEO_REG);

drivers/iommu/intel/iommu.c

4214

writeq(ecmd | (oa << DMA_ECMD_OA_SHIFT), iommu->reg + DMAR_ECMD_REG);

drivers/iommu/intel/iommu.c

4216

IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, readq,

drivers/iommu/intel/iommu.c

4226

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

443

static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)

drivers/iommu/intel/iommu.c

445

if (!iommu || iommu->drhd->ignored)

drivers/iommu/intel/iommu.c

464

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.c

486

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/iommu.c

520

iommu = NULL;

drivers/iommu/intel/iommu.c

522

if (iommu_is_dummy(iommu, dev))

drivers/iommu/intel/iommu.c

523

iommu = NULL;

drivers/iommu/intel/iommu.c

527

return iommu;

drivers/iommu/intel/iommu.c

53

#define rwbf_required(iommu) (rwbf_quirk || cap_rwbf((iommu)->cap))

drivers/iommu/intel/iommu.c

530

static void free_context_table(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

535

if (!iommu->root_entry)

drivers/iommu/intel/iommu.c

539

context = iommu_context_addr(iommu, i, 0, 0);

drivers/iommu/intel/iommu.c

543

if (!sm_supported(iommu))

drivers/iommu/intel/iommu.c

546

context = iommu_context_addr(iommu, i, 0x80, 0);

drivers/iommu/intel/iommu.c

551

iommu_free_pages(iommu->root_entry);

drivers/iommu/intel/iommu.c

552

iommu->root_entry = NULL;

drivers/iommu/intel/iommu.c

556

static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn,

drivers/iommu/intel/iommu.c

581

void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,

drivers/iommu/intel/iommu.c

593

pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);

drivers/iommu/intel/iommu.c

596

if (!iommu->root_entry) {

drivers/iommu/intel/iommu.c

600

rt_entry = &iommu->root_entry[bus];

drivers/iommu/intel/iommu.c

602

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

609

ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);

drivers/iommu/intel/iommu.c

619

if (!sm_supported(iommu)) {

drivers/iommu/intel/iommu.c

670

pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level);

drivers/iommu/intel/iommu.c

675

static int iommu_alloc_root_entry(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

679

root = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC, SZ_4K);

drivers/iommu/intel/iommu.c

682

iommu->name);

drivers/iommu/intel/iommu.c

686

__iommu_flush_cache(iommu, root, ROOT_SIZE);

drivers/iommu/intel/iommu.c

687

iommu->root_entry = root;

drivers/iommu/intel/iommu.c

692

static void iommu_set_root_entry(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

698

addr = virt_to_phys(iommu->root_entry);

drivers/iommu/intel/iommu.c

699

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

702

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

703

writeq(addr, iommu->reg + DMAR_RTADDR_REG);

drivers/iommu/intel/iommu.c

705

writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

708

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

711

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

717

if (cap_esrtps(iommu->cap))

drivers/iommu/intel/iommu.c

720

iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);

drivers/iommu/intel/iommu.c

721

if (sm_supported(iommu))

drivers/iommu/intel/iommu.c

722

qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);

drivers/iommu/intel/iommu.c

723

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

drivers/iommu/intel/iommu.c

726

void iommu_flush_write_buffer(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

731

if (!rwbf_quirk && !cap_rwbf(iommu->cap))

drivers/iommu/intel/iommu.c

734

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

735

writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

738

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

741

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

745

static void __iommu_flush_context(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.c

765

iommu->name, type);

drivers/iommu/intel/iommu.c

770

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

771

writeq(val, iommu->reg + DMAR_CCMD_REG);

drivers/iommu/intel/iommu.c

774

IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,

drivers/iommu/intel/iommu.c

777

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

780

void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.c

783

int tlb_offset = ecap_iotlb_offset(iommu->ecap);

drivers/iommu/intel/iommu.c

802

iommu->name, type);

drivers/iommu/intel/iommu.c

806

if (cap_write_drain(iommu->cap))

drivers/iommu/intel/iommu.c

809

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

812

writeq(val_iva, iommu->reg + tlb_offset);

drivers/iommu/intel/iommu.c

813

writeq(val, iommu->reg + tlb_offset + 8);

drivers/iommu/intel/iommu.c

816

IOMMU_WAIT_OP(iommu, tlb_offset + 8,

drivers/iommu/intel/iommu.c

819

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

832

struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.c

839

if (info->iommu == iommu && info->bus == bus &&

drivers/iommu/intel/iommu.c

917

iopf_queue_remove_device(info->iommu->iopf_queue, info->dev);

drivers/iommu/intel/iommu.c

928

static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

933

if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))

drivers/iommu/intel/iommu.c

936

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

937

pmen = readl(iommu->reg + DMAR_PMEN_REG);

drivers/iommu/intel/iommu.c

939

writel(pmen, iommu->reg + DMAR_PMEN_REG);

drivers/iommu/intel/iommu.c

942

IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,

drivers/iommu/intel/iommu.c

945

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

948

static void iommu_enable_translation(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

953

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

954

iommu->gcmd |= DMA_GCMD_TE;

drivers/iommu/intel/iommu.c

955

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

958

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

961

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/iommu.c

964

static void iommu_disable_translation(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

969

if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&

drivers/iommu/intel/iommu.c

970

(cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))

drivers/iommu/intel/iommu.c

973

raw_spin_lock_irqsave(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

974

iommu->gcmd &= ~DMA_GCMD_TE;

drivers/iommu/intel/iommu.c

975

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/iommu.c

978

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/iommu.c

981

raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

drivers/iommu/intel/iommu.c

984

static void disable_dmar_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

990

if (WARN_ON(!ida_is_empty(&iommu->domain_ida)))

drivers/iommu/intel/iommu.c

993

if (iommu->gcmd & DMA_GCMD_TE)

drivers/iommu/intel/iommu.c

994

iommu_disable_translation(iommu);

drivers/iommu/intel/iommu.c

997

static void free_dmar_iommu(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.c

999

if (iommu->copied_tables) {

drivers/iommu/intel/iommu.h

1040

static inline void qi_desc_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

1047

if (cap_write_drain(iommu->cap))

drivers/iommu/intel/iommu.h

1050

if (cap_read_drain(iommu->cap))

drivers/iommu/intel/iommu.h

1149

int dmar_enable_qi(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1150

void dmar_disable_qi(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1151

int dmar_reenable_qi(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1152

void qi_global_iec(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1154

void qi_flush_context(struct intel_iommu *iommu, u16 did,

drivers/iommu/intel/iommu.h

1156

void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

1158

void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/iommu.h

1161

void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);

drivers/iommu/intel/iommu.h

1163

void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,

drivers/iommu/intel/iommu.h

1169

void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu,

drivers/iommu/intel/iommu.h

1172

int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,

drivers/iommu/intel/iommu.h

1175

void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

1183

int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1184

void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1194

int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/iommu.h

1200

void iommu_flush_write_buffer(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1205

struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid);

drivers/iommu/intel/iommu.h

1217

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.h

1245

int intel_iommu_enable_prq(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1246

int intel_iommu_finish_prq(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1299

void intel_svm_check(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1303

static inline void intel_svm_check(struct intel_iommu *iommu) {}

drivers/iommu/intel/iommu.h

1326

struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,

drivers/iommu/intel/iommu.h

1345

int iommu_calculate_agaw(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1346

int iommu_calculate_max_sagaw(struct intel_iommu *iommu);

drivers/iommu/intel/iommu.h

1347

int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob);

drivers/iommu/intel/iommu.h

1349

static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

1351

return (iommu->ecmdcap[DMA_ECMD_ECCAP3] & DMA_ECMD_ECCAP3_ESSENTIAL) ==

drivers/iommu/intel/iommu.h

1358

static inline int iommu_calculate_agaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

1362

static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

358

#define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \

drivers/iommu/intel/iommu.h

362

sts = op(iommu->reg + offset); \

drivers/iommu/intel/iommu.h

467

#define qi_shift(iommu) (DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap))

drivers/iommu/intel/iommu.h

513

void (*flush_context)(struct intel_iommu *iommu, u16 did, u16 sid,

drivers/iommu/intel/iommu.h

515

void (*flush_iotlb)(struct intel_iommu *iommu, u16 did, u64 addr,

drivers/iommu/intel/iommu.h

531

#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))

drivers/iommu/intel/iommu.h

532

#define pasid_supported(iommu) (sm_supported(iommu) && \

drivers/iommu/intel/iommu.h

533

ecap_pasid((iommu)->ecap))

drivers/iommu/intel/iommu.h

534

#define ssads_supported(iommu) (sm_supported(iommu) && \

drivers/iommu/intel/iommu.h

535

ecap_slads((iommu)->ecap) && \

drivers/iommu/intel/iommu.h

536

ecap_smpwc(iommu->ecap))

drivers/iommu/intel/iommu.h

537

#define nested_supported(iommu) (sm_supported(iommu) && \

drivers/iommu/intel/iommu.h

538

ecap_nest((iommu)->ecap))

drivers/iommu/intel/iommu.h

572

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.h

595

struct pt_iommu iommu;

drivers/iommu/intel/iommu.h

644

PT_IOMMU_CHECK_DOMAIN(struct dmar_domain, iommu, domain);

drivers/iommu/intel/iommu.h

645

PT_IOMMU_CHECK_DOMAIN(struct dmar_domain, sspt.iommu, domain);

drivers/iommu/intel/iommu.h

646

PT_IOMMU_CHECK_DOMAIN(struct dmar_domain, fspt.iommu, domain);

drivers/iommu/intel/iommu.h

659

struct intel_iommu *iommu;

drivers/iommu/intel/iommu.h

729

struct iommu_device iommu; /* IOMMU core code handle */

drivers/iommu/intel/iommu.h

757

struct intel_iommu *iommu; /* IOMMU used by this device */

drivers/iommu/intel/iommu.h

777

struct intel_iommu *iommu, void *addr, int size)

drivers/iommu/intel/iommu.h

779

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/iommu.h

809

domain_id_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

812

xa_load(&domain->iommu_array, iommu->seq_id);

drivers/iommu/intel/iommu.h

818

iommu_domain_did(struct iommu_domain *domain, struct intel_iommu *iommu)

drivers/iommu/intel/iommu.h

823

return domain_id_iommu(to_dmar_domain(domain), iommu);

drivers/iommu/intel/iommu.h

968

static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.h

970

if (!iommu->copied_tables)

drivers/iommu/intel/iommu.h

973

return test_bit(((long)bus << 8) | devfn, iommu->copied_tables);

drivers/iommu/intel/iommu.h

977

set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.h

979

set_bit(((long)bus << 8) | devfn, iommu->copied_tables);

drivers/iommu/intel/iommu.h

983

clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)

drivers/iommu/intel/iommu.h

985

clear_bit(((long)bus << 8) | devfn, iommu->copied_tables);

drivers/iommu/intel/irq_remapping.c

101

iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;

drivers/iommu/intel/irq_remapping.c

1013

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/irq_remapping.c

1018

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

1019

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

1022

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1036

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/irq_remapping.c

1038

for_each_iommu(iommu, drhd)

drivers/iommu/intel/irq_remapping.c

1039

if (iommu->qi)

drivers/iommu/intel/irq_remapping.c

104

static int alloc_irte(struct intel_iommu *iommu,

drivers/iommu/intel/irq_remapping.c

1040

dmar_reenable_qi(iommu);

drivers/iommu/intel/irq_remapping.c

1045

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

1046

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

1050

iommu_set_irq_remapping(iommu, eim);

drivers/iommu/intel/irq_remapping.c

1051

iommu_enable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

107

struct ir_table *table = iommu->ir_table;

drivers/iommu/intel/irq_remapping.c

120

if (mask > ecap_max_handle_mask(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

123

ecap_max_handle_mask(iommu->ecap));

drivers/iommu/intel/irq_remapping.c

131

pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

133

irq_iommu->iommu = iommu;

drivers/iommu/intel/irq_remapping.c

1412

struct intel_iommu *iommu = domain->host_data;

drivers/iommu/intel/irq_remapping.c

1419

if (!info || !iommu)

drivers/iommu/intel/irq_remapping.c

143

static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)

drivers/iommu/intel/irq_remapping.c

1433

index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs);

drivers/iommu/intel/irq_remapping.c

1511

struct intel_iommu *iommu = NULL;

drivers/iommu/intel/irq_remapping.c

1514

iommu = map_ioapic_to_iommu(fwspec->param[0]);

drivers/iommu/intel/irq_remapping.c

1516

iommu = map_hpet_to_iommu(fwspec->param[0]);

drivers/iommu/intel/irq_remapping.c

1518

return iommu && d == iommu->ir_domain;

drivers/iommu/intel/irq_remapping.c

153

return qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/irq_remapping.c

1540

static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

1545

if (eim && !ecap_eim_support(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

1547

iommu->reg_phys, iommu->ecap);

drivers/iommu/intel/irq_remapping.c

1551

if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) {

drivers/iommu/intel/irq_remapping.c

1553

iommu->reg_phys);

drivers/iommu/intel/irq_remapping.c

1560

ret = intel_setup_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1563

iommu->name);

drivers/iommu/intel/irq_remapping.c

1564

intel_teardown_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1565

ir_remove_ioapic_hpet_scope(iommu);

drivers/iommu/intel/irq_remapping.c

1567

iommu_enable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1576

struct intel_iommu *iommu = dmaru->iommu;

drivers/iommu/intel/irq_remapping.c

1580

if (iommu == NULL)

drivers/iommu/intel/irq_remapping.c

1582

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

1585

!cap_pi_support(iommu->cap))

drivers/iommu/intel/irq_remapping.c

1589

if (!iommu->ir_table)

drivers/iommu/intel/irq_remapping.c

159

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

1590

ret = dmar_ir_add(dmaru, iommu);

drivers/iommu/intel/irq_remapping.c

1592

if (iommu->ir_table) {

drivers/iommu/intel/irq_remapping.c

1593

if (!bitmap_empty(iommu->ir_table->bitmap,

drivers/iommu/intel/irq_remapping.c

1597

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1598

intel_teardown_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

1599

ir_remove_ioapic_hpet_scope(iommu);

drivers/iommu/intel/irq_remapping.c

169

iommu = irq_iommu->iommu;

drivers/iommu/intel/irq_remapping.c

172

irte = &iommu->ir_table->base[index];

drivers/iommu/intel/irq_remapping.c

187

__iommu_flush_cache(iommu, irte, sizeof(*irte));

drivers/iommu/intel/irq_remapping.c

189

rc = qi_flush_iec(iommu, index, 0);

drivers/iommu/intel/irq_remapping.c

201

if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)

drivers/iommu/intel/irq_remapping.c

202

return ir_hpet[i].iommu;

drivers/iommu/intel/irq_remapping.c

212

if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)

drivers/iommu/intel/irq_remapping.c

213

return ir_ioapic[i].iommu;

drivers/iommu/intel/irq_remapping.c

222

return drhd ? drhd->iommu->ir_domain : NULL;

drivers/iommu/intel/irq_remapping.c

228

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

234

iommu = irq_iommu->iommu;

drivers/iommu/intel/irq_remapping.c

237

start = iommu->ir_table->base + index;

drivers/iommu/intel/irq_remapping.c

244

bitmap_release_region(iommu->ir_table->bitmap, index,

drivers/iommu/intel/irq_remapping.c

247

return qi_flush_iec(iommu, index, irq_iommu->irte_mask);

drivers/iommu/intel/irq_remapping.c

30

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

306

if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) {

drivers/iommu/intel/irq_remapping.c

331

if (ir_hpet[i].iommu && ir_hpet[i].id == id) {

drivers/iommu/intel/irq_remapping.c

37

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

416

static int iommu_load_old_irte(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

425

irta = readq(iommu->reg + DMAR_IRTA_REG);

drivers/iommu/intel/irq_remapping.c

439

memcpy(iommu->ir_table->base, old_ir_table, size);

drivers/iommu/intel/irq_remapping.c

44

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

441

__iommu_flush_cache(iommu, iommu->ir_table->base, size);

drivers/iommu/intel/irq_remapping.c

448

if (iommu->ir_table->base[i].present)

drivers/iommu/intel/irq_remapping.c

449

bitmap_set(iommu->ir_table->bitmap, i, 1);

drivers/iommu/intel/irq_remapping.c

458

static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)

drivers/iommu/intel/irq_remapping.c

464

addr = virt_to_phys((void *)iommu->ir_table->base);

drivers/iommu/intel/irq_remapping.c

466

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

469

iommu->reg + DMAR_IRTA_REG);

drivers/iommu/intel/irq_remapping.c

472

writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

474

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

476

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

482

if (!cap_esirtps(iommu->cap))

drivers/iommu/intel/irq_remapping.c

483

qi_global_iec(iommu);

drivers/iommu/intel/irq_remapping.c

486

static void iommu_enable_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

491

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

494

iommu->gcmd |= DMA_GCMD_IRE;

drivers/iommu/intel/irq_remapping.c

495

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

496

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

501

iommu->gcmd &= ~DMA_GCMD_CFI;

drivers/iommu/intel/irq_remapping.c

502

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

503

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

517

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

520

static int intel_setup_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

527

.host_data = iommu,

drivers/iommu/intel/irq_remapping.c

533

if (iommu->ir_table)

drivers/iommu/intel/irq_remapping.c

542

iommu_alloc_pages_node_sz(iommu->node, GFP_KERNEL, SZ_1M);

drivers/iommu/intel/irq_remapping.c

544

pr_err("IR%d: failed to allocate 1M of pages\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

550

pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

554

info.fwnode = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

558

iommu->ir_domain = msi_create_parent_irq_domain(&info, &dmar_msi_parent_ops);

drivers/iommu/intel/irq_remapping.c

559

if (!iommu->ir_domain) {

drivers/iommu/intel/irq_remapping.c

560

pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

566

iommu->ir_table = ir_table;

drivers/iommu/intel/irq_remapping.c

572

if (!iommu->qi) {

drivers/iommu/intel/irq_remapping.c

576

dmar_fault(-1, iommu);

drivers/iommu/intel/irq_remapping.c

577

dmar_disable_qi(iommu);

drivers/iommu/intel/irq_remapping.c

579

if (dmar_enable_qi(iommu)) {

drivers/iommu/intel/irq_remapping.c

585

init_ir_status(iommu);

drivers/iommu/intel/irq_remapping.c

587

if (ir_pre_enabled(iommu)) {

drivers/iommu/intel/irq_remapping.c

590

iommu->name);

drivers/iommu/intel/irq_remapping.c

591

clear_ir_pre_enabled(iommu);

drivers/iommu/intel/irq_remapping.c

592

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

593

} else if (iommu_load_old_irte(iommu))

drivers/iommu/intel/irq_remapping.c

595

iommu->name);

drivers/iommu/intel/irq_remapping.c

598

iommu->name);

drivers/iommu/intel/irq_remapping.c

601

iommu_set_irq_remapping(iommu, eim_mode);

drivers/iommu/intel/irq_remapping.c

606

irq_domain_remove(iommu->ir_domain);

drivers/iommu/intel/irq_remapping.c

607

iommu->ir_domain = NULL;

drivers/iommu/intel/irq_remapping.c

617

iommu->ir_table = NULL;

drivers/iommu/intel/irq_remapping.c

622

static void intel_teardown_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

626

if (iommu && iommu->ir_table) {

drivers/iommu/intel/irq_remapping.c

627

if (iommu->ir_domain) {

drivers/iommu/intel/irq_remapping.c

628

fn = iommu->ir_domain->fwnode;

drivers/iommu/intel/irq_remapping.c

630

irq_domain_remove(iommu->ir_domain);

drivers/iommu/intel/irq_remapping.c

632

iommu->ir_domain = NULL;

drivers/iommu/intel/irq_remapping.c

634

iommu_free_pages(iommu->ir_table->base);

drivers/iommu/intel/irq_remapping.c

635

bitmap_free(iommu->ir_table->bitmap);

drivers/iommu/intel/irq_remapping.c

636

kfree(iommu->ir_table);

drivers/iommu/intel/irq_remapping.c

637

iommu->ir_table = NULL;

drivers/iommu/intel/irq_remapping.c

644

static void iommu_disable_irq_remapping(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

649

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

656

if (!cap_esirtps(iommu->cap))

drivers/iommu/intel/irq_remapping.c

657

qi_global_iec(iommu);

drivers/iommu/intel/irq_remapping.c

659

raw_spin_lock_irqsave(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

661

sts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/irq_remapping.c

665

iommu->gcmd &= ~DMA_GCMD_IRE;

drivers/iommu/intel/irq_remapping.c

666

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/intel/irq_remapping.c

668

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

drivers/iommu/intel/irq_remapping.c

672

raw_spin_unlock_irqrestore(&iommu->register_lock, flags);

drivers/iommu/intel/irq_remapping.c

687

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

689

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

690

if (ecap_ir_support(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

691

iommu_disable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

692

intel_teardown_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

703

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

728

for_each_iommu(iommu, drhd)

drivers/iommu/intel/irq_remapping.c

729

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

741

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

742

if (eim && !ecap_eim_support(iommu->ecap)) {

drivers/iommu/intel/irq_remapping.c

743

pr_info("%s does not support EIM\n", iommu->name);

drivers/iommu/intel/irq_remapping.c

753

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

754

if (intel_setup_irq_remapping(iommu)) {

drivers/iommu/intel/irq_remapping.c

756

iommu->name);

drivers/iommu/intel/irq_remapping.c

774

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

788

for_each_iommu(iommu, drhd)

drivers/iommu/intel/irq_remapping.c

789

if (!cap_pi_support(iommu->cap)) {

drivers/iommu/intel/irq_remapping.c

800

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

806

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

807

if (!ir_pre_enabled(iommu))

drivers/iommu/intel/irq_remapping.c

808

iommu_enable_irq_remapping(iommu);

drivers/iommu/intel/irq_remapping.c

81

static void iommu_disable_irq_remapping(struct intel_iommu *iommu);

drivers/iommu/intel/irq_remapping.c

829

struct intel_iommu *iommu,

drivers/iommu/intel/irq_remapping.c

85

static bool ir_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

852

if (ir_hpet[count].iommu == iommu &&

drivers/iommu/intel/irq_remapping.c

855

else if (ir_hpet[count].iommu == NULL && free == -1)

drivers/iommu/intel/irq_remapping.c

863

ir_hpet[free].iommu = iommu;

drivers/iommu/intel/irq_remapping.c

87

return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);

drivers/iommu/intel/irq_remapping.c

874

struct intel_iommu *iommu,

drivers/iommu/intel/irq_remapping.c

897

if (ir_ioapic[count].iommu == iommu &&

drivers/iommu/intel/irq_remapping.c

90

static void clear_ir_pre_enabled(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

900

else if (ir_ioapic[count].iommu == NULL && free == -1)

drivers/iommu/intel/irq_remapping.c

910

ir_ioapic[free].iommu = iommu;

drivers/iommu/intel/irq_remapping.c

913

scope->enumeration_id, drhd->address, iommu->seq_id);

drivers/iommu/intel/irq_remapping.c

919

struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

92

iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;

drivers/iommu/intel/irq_remapping.c

933

ret = ir_parse_one_ioapic_scope(scope, iommu, drhd);

drivers/iommu/intel/irq_remapping.c

935

ret = ir_parse_one_hpet_scope(scope, iommu, drhd);

drivers/iommu/intel/irq_remapping.c

942

static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

947

if (ir_hpet[i].iommu == iommu)

drivers/iommu/intel/irq_remapping.c

948

ir_hpet[i].iommu = NULL;

drivers/iommu/intel/irq_remapping.c

95

static void init_ir_status(struct intel_iommu *iommu)

drivers/iommu/intel/irq_remapping.c

951

if (ir_ioapic[i].iommu == iommu)

drivers/iommu/intel/irq_remapping.c

952

ir_ioapic[i].iommu = NULL;

drivers/iommu/intel/irq_remapping.c

962

struct intel_iommu *iommu;

drivers/iommu/intel/irq_remapping.c

966

for_each_iommu(iommu, drhd) {

drivers/iommu/intel/irq_remapping.c

969

if (!ecap_ir_support(iommu->ecap))

drivers/iommu/intel/irq_remapping.c

972

ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu);

drivers/iommu/intel/irq_remapping.c

99

gsts = readl(iommu->reg + DMAR_GSTS_REG);

drivers/iommu/intel/nested.c

134

static int domain_setup_nested(struct intel_iommu *iommu,

drivers/iommu/intel/nested.c

140

intel_pasid_tear_down_entry(iommu, dev, pasid, false);

drivers/iommu/intel/nested.c

142

return intel_pasid_setup_nested(iommu, dev, pasid, domain);

drivers/iommu/intel/nested.c

151

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/nested.c

155

if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))

drivers/iommu/intel/nested.c

158

if (context_copied(iommu, info->bus, info->devfn))

drivers/iommu/intel/nested.c

173

ret = domain_setup_nested(iommu, dmar_domain, dev, pasid, old);

drivers/iommu/intel/nested.c

202

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/nested.c

207

if (!nested_supported(iommu) || flags & ~IOMMU_HWPT_ALLOC_PASID)

drivers/iommu/intel/nested.c

26

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/nested.c

43

ret = domain_attach_iommu(dmar_domain, iommu);

drivers/iommu/intel/nested.c

57

ret = intel_pasid_setup_nested(iommu, dev,

drivers/iommu/intel/nested.c

74

domain_detach_iommu(dmar_domain, iommu);

drivers/iommu/intel/pasid.c

151

entries = iommu_alloc_pages_node_sz(info->iommu->node,

drivers/iommu/intel/pasid.c

156

if (!ecap_coherent(info->iommu->ecap))

drivers/iommu/intel/pasid.c

171

if (!ecap_coherent(info->iommu->ecap))

drivers/iommu/intel/pasid.c

197

pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

208

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/pasid.c

212

devtlb_invalidation_with_pasid(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

236

qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);

drivers/iommu/intel/pasid.c

238

qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);

drivers/iommu/intel/pasid.c

241

void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.c

247

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

250

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

257

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

268

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

277

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

279

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

282

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

285

qi_flush_piotlb_all(iommu, did, pasid);

drivers/iommu/intel/pasid.c

287

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c

289

devtlb_invalidation_with_pasid(iommu, dev, pasid);

drivers/iommu/intel/pasid.c

291

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

302

static void pasid_flush_caches(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

306

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

309

if (cap_caching_mode(iommu->cap)) {

drivers/iommu/intel/pasid.c

310

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

311

qi_flush_piotlb_all(iommu, did, pasid);

drivers/iommu/intel/pasid.c

313

iommu_flush_write_buffer(iommu);

drivers/iommu/intel/pasid.c

325

static void intel_pasid_flush_present(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

330

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

344

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

345

qi_flush_piotlb_all(iommu, did, pasid);

drivers/iommu/intel/pasid.c

347

devtlb_invalidation_with_pasid(iommu, dev, pasid);

drivers/iommu/intel/pasid.c

354

static void pasid_pte_config_first_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

359

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

373

pasid_set_address_width(pte, iommu->agaw);

drivers/iommu/intel/pasid.c

381

int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.c

387

if (!ecap_flts(iommu->ecap)) {

drivers/iommu/intel/pasid.c

389

iommu->name);

drivers/iommu/intel/pasid.c

393

if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/pasid.c

395

iommu->name);

drivers/iommu/intel/pasid.c

399

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

402

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

407

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

411

pasid_pte_config_first_level(iommu, pte, fsptptr, did, flags);

drivers/iommu/intel/pasid.c

413

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

415

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

423

static void pasid_pte_config_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

429

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

446

int intel_pasid_setup_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

458

if (!ecap_slts(iommu->ecap)) {

drivers/iommu/intel/pasid.c

460

iommu->name);

drivers/iommu/intel/pasid.c

464

did = domain_id_iommu(domain, iommu);

drivers/iommu/intel/pasid.c

466

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

469

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

474

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

478

pasid_pte_config_second_level(iommu, pte, domain, did);

drivers/iommu/intel/pasid.c

479

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

481

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

489

int intel_pasid_setup_dirty_tracking(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

496

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

500

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

510

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

519

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

527

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

529

if (!ecap_coherent(iommu->ecap))

drivers/iommu/intel/pasid.c

546

pasid_cache_invalidation_with_pasid(iommu, did, pasid);

drivers/iommu/intel/pasid.c

548

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c

550

devtlb_invalidation_with_pasid(iommu, dev, pasid);

drivers/iommu/intel/pasid.c

558

static void pasid_pte_config_pass_through(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

561

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

565

pasid_set_address_width(pte, iommu->agaw);

drivers/iommu/intel/pasid.c

568

pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));

drivers/iommu/intel/pasid.c

572

int intel_pasid_setup_pass_through(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

578

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

581

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

586

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

590

pasid_pte_config_pass_through(iommu, pte, did);

drivers/iommu/intel/pasid.c

591

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

593

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

601

void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

607

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

610

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

616

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

618

intel_pasid_flush_present(iommu, dev, pasid, did, pte);

drivers/iommu/intel/pasid.c

621

static void pasid_pte_config_nested(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.c

629

lockdep_assert_held(&iommu->lock);

drivers/iommu/intel/pasid.c

63

dir = iommu_alloc_pages_node_sz(info->iommu->node, GFP_KERNEL,

drivers/iommu/intel/pasid.c

675

int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.c

680

u16 did = domain_id_iommu(domain, iommu);

drivers/iommu/intel/pasid.c

688

if (!cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/pasid.c

700

if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) {

drivers/iommu/intel/pasid.c

702

iommu->name);

drivers/iommu/intel/pasid.c

706

if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) {

drivers/iommu/intel/pasid.c

708

iommu->name);

drivers/iommu/intel/pasid.c

712

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

715

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

719

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

723

pasid_pte_config_nested(iommu, pte, s1_cfg, s2_domain, did);

drivers/iommu/intel/pasid.c

724

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

726

pasid_flush_caches(iommu, pte, pasid, did);

drivers/iommu/intel/pasid.c

739

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

74

if (!ecap_coherent(info->iommu->ecap))

drivers/iommu/intel/pasid.c

743

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

744

context = iommu_context_addr(iommu, bus, devfn, false);

drivers/iommu/intel/pasid.c

746

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

752

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

753

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

756

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

803

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

821

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

829

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

832

spin_lock(&iommu->lock);

drivers/iommu/intel/pasid.c

833

context = iommu_context_addr(iommu, bus, devfn, true);

drivers/iommu/intel/pasid.c

835

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

839

if (context_present(context) && !context_copied(iommu, bus, devfn)) {

drivers/iommu/intel/pasid.c

840

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

844

if (context_copied(iommu, bus, devfn)) {

drivers/iommu/intel/pasid.c

846

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

857

iommu->flush.flush_context(iommu, 0,

drivers/iommu/intel/pasid.c

861

qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);

drivers/iommu/intel/pasid.c

862

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

drivers/iommu/intel/pasid.c

863

devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID);

drivers/iommu/intel/pasid.c

866

__iommu_flush_cache(iommu, context, sizeof(*context));

drivers/iommu/intel/pasid.c

873

clear_context_copied(iommu, bus, devfn);

drivers/iommu/intel/pasid.c

877

spin_unlock(&iommu->lock);

drivers/iommu/intel/pasid.c

885

if (cap_caching_mode(iommu->cap)) {

drivers/iommu/intel/pasid.c

886

iommu->flush.flush_context(iommu, 0,

drivers/iommu/intel/pasid.c

890

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c

939

qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn),

drivers/iommu/intel/pasid.c

962

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/pasid.c

970

iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn),

drivers/iommu/intel/pasid.c

978

if (!sm_supported(iommu)) {

drivers/iommu/intel/pasid.c

979

iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.h

306

int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.h

309

int intel_pasid_setup_second_level(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

312

int intel_pasid_setup_dirty_tracking(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

315

int intel_pasid_setup_pass_through(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

317

int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/pasid.h

319

void intel_pasid_tear_down_entry(struct intel_iommu *iommu,

drivers/iommu/intel/pasid.h

322

void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,

drivers/iommu/intel/perf.c

116

void dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size)

drivers/iommu/intel/perf.c

118

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

130

if (!dmar_latency_enabled(iommu, i))

drivers/iommu/intel/perf.c

18

bool dmar_latency_enabled(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.c

20

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

25

int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.c

31

if (dmar_latency_enabled(iommu, type))

drivers/iommu/intel/perf.c

35

if (!iommu->perf_statistic) {

drivers/iommu/intel/perf.c

36

iommu->perf_statistic = kzalloc_objs(*lstat, DMAR_LATENCY_NUM,

drivers/iommu/intel/perf.c

38

if (!iommu->perf_statistic) {

drivers/iommu/intel/perf.c

44

lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

57

void dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.c

59

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

62

if (!dmar_latency_enabled(iommu, type))

drivers/iommu/intel/perf.c

70

void dmar_latency_update(struct intel_iommu *iommu, enum latency_type type, u64 latency)

drivers/iommu/intel/perf.c

72

struct latency_statistic *lstat = iommu->perf_statistic;

drivers/iommu/intel/perf.c

76

if (!dmar_latency_enabled(iommu, type))

drivers/iommu/intel/perf.h

38

int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type);

drivers/iommu/intel/perf.h

39

void dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type);

drivers/iommu/intel/perf.h

40

bool dmar_latency_enabled(struct intel_iommu *iommu, enum latency_type type);

drivers/iommu/intel/perf.h

41

void dmar_latency_update(struct intel_iommu *iommu, enum latency_type type,

drivers/iommu/intel/perf.h

43

void dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size);

drivers/iommu/intel/perf.h

46

dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.h

52

dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.h

57

dmar_latency_enabled(struct intel_iommu *iommu, enum latency_type type)

drivers/iommu/intel/perf.h

63

dmar_latency_update(struct intel_iommu *iommu, enum latency_type type, u64 latency)

drivers/iommu/intel/perf.h

68

dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size)

drivers/iommu/intel/perfmon.c

327

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

356

ecmd_submit_sync(iommu, DMA_ECMD_ENABLE, hwc->idx, 0);

drivers/iommu/intel/perfmon.c

364

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

368

ecmd_submit_sync(iommu, DMA_ECMD_DISABLE, hwc->idx, 0);

drivers/iommu/intel/perfmon.c

476

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

478

ecmd_submit_sync(iommu, DMA_ECMD_UNFREEZE, 0, 0);

drivers/iommu/intel/perfmon.c

484

struct intel_iommu *iommu = iommu_pmu->iommu;

drivers/iommu/intel/perfmon.c

486

ecmd_submit_sync(iommu, DMA_ECMD_FREEZE, 0, 0);

drivers/iommu/intel/perfmon.c

519

struct intel_iommu *iommu = dev_id;

drivers/iommu/intel/perfmon.c

521

if (!readl(iommu->reg + DMAR_PERFINTRSTS_REG))

drivers/iommu/intel/perfmon.c

524

iommu_pmu_counter_overflow(iommu->pmu);

drivers/iommu/intel/perfmon.c

527

writel(DMA_PERFINTRSTS_PIS, iommu->reg + DMAR_PERFINTRSTS_REG);

drivers/iommu/intel/perfmon.c

532

static int __iommu_pmu_register(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

534

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

536

iommu_pmu->pmu.name = iommu->name;

drivers/iommu/intel/perfmon.c

556

get_perf_reg_address(struct intel_iommu *iommu, u32 offset)

drivers/iommu/intel/perfmon.c

558

u32 off = readl(iommu->reg + offset);

drivers/iommu/intel/perfmon.c

560

return iommu->reg + off;

drivers/iommu/intel/perfmon.c

563

int alloc_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

570

if (!ecap_pms(iommu->ecap))

drivers/iommu/intel/perfmon.c

574

if (!cap_ecmds(iommu->cap))

drivers/iommu/intel/perfmon.c

577

perfcap = readq(iommu->reg + DMAR_PERFCAP_REG);

drivers/iommu/intel/perfmon.c

591

if (!ecmd_has_pmu_essential(iommu))

drivers/iommu/intel/perfmon.c

620

pcap = readq(iommu->reg + DMAR_PERFEVNTCAP_REG +

drivers/iommu/intel/perfmon.c

644

iommu_pmu->cfg_reg = get_perf_reg_address(iommu, DMAR_PERFCFGOFF_REG);

drivers/iommu/intel/perfmon.c

645

iommu_pmu->cntr_reg = get_perf_reg_address(iommu, DMAR_PERFCNTROFF_REG);

drivers/iommu/intel/perfmon.c

646

iommu_pmu->overflow = get_perf_reg_address(iommu, DMAR_PERFOVFOFF_REG);

drivers/iommu/intel/perfmon.c

690

iommu_pmu->iommu = iommu;

drivers/iommu/intel/perfmon.c

691

iommu->pmu = iommu_pmu;

drivers/iommu/intel/perfmon.c

707

void free_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

709

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

723

iommu->pmu = NULL;

drivers/iommu/intel/perfmon.c

726

static int iommu_pmu_set_interrupt(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

728

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

731

irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PERF + iommu->seq_id, iommu->node, iommu);

drivers/iommu/intel/perfmon.c

735

snprintf(iommu_pmu->irq_name, sizeof(iommu_pmu->irq_name), "dmar%d-perf", iommu->seq_id);

drivers/iommu/intel/perfmon.c

737

iommu->perf_irq = irq;

drivers/iommu/intel/perfmon.c

739

IRQF_ONESHOT, iommu_pmu->irq_name, iommu);

drivers/iommu/intel/perfmon.c

742

iommu->perf_irq = 0;

drivers/iommu/intel/perfmon.c

748

static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

750

if (!iommu->perf_irq)

drivers/iommu/intel/perfmon.c

753

free_irq(iommu->perf_irq, iommu);

drivers/iommu/intel/perfmon.c

754

dmar_free_hwirq(iommu->perf_irq);

drivers/iommu/intel/perfmon.c

755

iommu->perf_irq = 0;

drivers/iommu/intel/perfmon.c

758

void iommu_pmu_register(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

760

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

765

if (__iommu_pmu_register(iommu))

drivers/iommu/intel/perfmon.c

769

if (iommu_pmu_set_interrupt(iommu))

drivers/iommu/intel/perfmon.c

777

pr_err("Failed to register PMU for iommu (seq_id = %d)\n", iommu->seq_id);

drivers/iommu/intel/perfmon.c

778

free_iommu_pmu(iommu);

drivers/iommu/intel/perfmon.c

781

void iommu_pmu_unregister(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.c

783

struct iommu_pmu *iommu_pmu = iommu->pmu;

drivers/iommu/intel/perfmon.c

788

iommu_pmu_unset_interrupt(iommu);

drivers/iommu/intel/perfmon.h

39

int alloc_iommu_pmu(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

40

void free_iommu_pmu(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

41

void iommu_pmu_register(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

42

void iommu_pmu_unregister(struct intel_iommu *iommu);

drivers/iommu/intel/perfmon.h

45

alloc_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.h

51

free_iommu_pmu(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.h

56

iommu_pmu_register(struct intel_iommu *iommu)

drivers/iommu/intel/perfmon.h

61

iommu_pmu_unregister(struct intel_iommu *iommu)

drivers/iommu/intel/prq.c

112

qi_desc_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH, &desc[1]);

drivers/iommu/intel/prq.c

121

reinit_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

122

qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);

drivers/iommu/intel/prq.c

123

if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {

drivers/iommu/intel/prq.c

124

wait_for_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

137

static void handle_bad_prq_event(struct intel_iommu *iommu,

drivers/iommu/intel/prq.c

143

iommu->name, ((unsigned long long *)req)[0],

drivers/iommu/intel/prq.c

156

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/prq.c

175

static void intel_prq_report(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/prq.c

199

struct intel_iommu *iommu = d;

drivers/iommu/intel/prq.c

209

writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);

drivers/iommu/intel/prq.c

211

tail = readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

212

head = readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

215

req = &iommu->prq[head / sizeof(*req)];

drivers/iommu/intel/prq.c

220

iommu->name);

drivers/iommu/intel/prq.c

222

handle_bad_prq_event(iommu, req, QI_RESP_INVALID);

drivers/iommu/intel/prq.c

228

iommu->name);

drivers/iommu/intel/prq.c

234

iommu->name);

drivers/iommu/intel/prq.c

246

mutex_lock(&iommu->iopf_lock);

drivers/iommu/intel/prq.c

247

dev = device_rbtree_find(iommu, req->rid);

drivers/iommu/intel/prq.c

249

mutex_unlock(&iommu->iopf_lock);

drivers/iommu/intel/prq.c

253

intel_prq_report(iommu, dev, req);

drivers/iommu/intel/prq.c

254

trace_prq_report(iommu, dev, req->qw_0, req->qw_1,

drivers/iommu/intel/prq.c

256

iommu->prq_seq_number++);

drivers/iommu/intel/prq.c

257

mutex_unlock(&iommu->iopf_lock);

drivers/iommu/intel/prq.c

262

writeq(tail, iommu->reg + DMAR_PQH_REG);

drivers/iommu/intel/prq.c

268

if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {

drivers/iommu/intel/prq.c

270

iommu->name);

drivers/iommu/intel/prq.c

271

head = readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

272

tail = readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

274

iopf_queue_discard_partial(iommu->iopf_queue);

drivers/iommu/intel/prq.c

275

writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG);

drivers/iommu/intel/prq.c

277

iommu->name);

drivers/iommu/intel/prq.c

281

if (!completion_done(&iommu->prq_complete))

drivers/iommu/intel/prq.c

282

complete(&iommu->prq_complete);

drivers/iommu/intel/prq.c

287

int intel_iommu_enable_prq(struct intel_iommu *iommu)

drivers/iommu/intel/prq.c

292

iommu->prq =

drivers/iommu/intel/prq.c

293

iommu_alloc_pages_node_sz(iommu->node, GFP_KERNEL, PRQ_SIZE);

drivers/iommu/intel/prq.c

294

if (!iommu->prq) {

drivers/iommu/intel/prq.c

296

iommu->name);

drivers/iommu/intel/prq.c

300

irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu);

drivers/iommu/intel/prq.c

303

iommu->name);

drivers/iommu/intel/prq.c

307

iommu->pr_irq = irq;

drivers/iommu/intel/prq.c

309

snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name),

drivers/iommu/intel/prq.c

310

"dmar%d-iopfq", iommu->seq_id);

drivers/iommu/intel/prq.c

311

iopfq = iopf_queue_alloc(iommu->iopfq_name);

drivers/iommu/intel/prq.c

313

pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu->name);

drivers/iommu/intel/prq.c

317

iommu->iopf_queue = iopfq;

drivers/iommu/intel/prq.c

319

snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id);

drivers/iommu/intel/prq.c

322

iommu->prq_name, iommu);

drivers/iommu/intel/prq.c

325

iommu->name);

drivers/iommu/intel/prq.c

328

writeq(0ULL, iommu->reg + DMAR_PQH_REG);

drivers/iommu/intel/prq.c

329

writeq(0ULL, iommu->reg + DMAR_PQT_REG);

drivers/iommu/intel/prq.c

330

writeq(virt_to_phys(iommu->prq) | PRQ_ORDER, iommu->reg + DMAR_PQA_REG);

drivers/iommu/intel/prq.c

332

init_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

337

iopf_queue_free(iommu->iopf_queue);

drivers/iommu/intel/prq.c

338

iommu->iopf_queue = NULL;

drivers/iommu/intel/prq.c

341

iommu->pr_irq = 0;

drivers/iommu/intel/prq.c

343

iommu_free_pages(iommu->prq);

drivers/iommu/intel/prq.c

344

iommu->prq = NULL;

drivers/iommu/intel/prq.c

349

int intel_iommu_finish_prq(struct intel_iommu *iommu)

drivers/iommu/intel/prq.c

351

writeq(0ULL, iommu->reg + DMAR_PQH_REG);

drivers/iommu/intel/prq.c

352

writeq(0ULL, iommu->reg + DMAR_PQT_REG);

drivers/iommu/intel/prq.c

353

writeq(0ULL, iommu->reg + DMAR_PQA_REG);

drivers/iommu/intel/prq.c

355

if (iommu->pr_irq) {

drivers/iommu/intel/prq.c

356

free_irq(iommu->pr_irq, iommu);

drivers/iommu/intel/prq.c

357

dmar_free_hwirq(iommu->pr_irq);

drivers/iommu/intel/prq.c

358

iommu->pr_irq = 0;

drivers/iommu/intel/prq.c

361

if (iommu->iopf_queue) {

drivers/iommu/intel/prq.c

362

iopf_queue_free(iommu->iopf_queue);

drivers/iommu/intel/prq.c

363

iommu->iopf_queue = NULL;

drivers/iommu/intel/prq.c

366

iommu_free_pages(iommu->prq);

drivers/iommu/intel/prq.c

367

iommu->prq = NULL;

drivers/iommu/intel/prq.c

376

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/prq.c

395

qi_submit_sync(iommu, &desc, 1, 0);

drivers/iommu/intel/prq.c

64

struct intel_iommu *iommu;

drivers/iommu/intel/prq.c

73

iommu = info->iommu;

drivers/iommu/intel/prq.c

76

did = domain ? domain_id_iommu(domain, iommu) : FLPT_DEFAULT_DID;

drivers/iommu/intel/prq.c

83

reinit_completion(&iommu->prq_complete);

drivers/iommu/intel/prq.c

84

tail = readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

85

head = readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;

drivers/iommu/intel/prq.c

89

req = &iommu->prq[head / sizeof(*req)];

drivers/iommu/intel/prq.c

97

wait_for_completion(&iommu->prq_complete);

drivers/iommu/intel/svm.c

116

struct intel_iommu *iommu;

drivers/iommu/intel/svm.c

121

iommu = info->iommu;

drivers/iommu/intel/svm.c

122

if (!iommu)

drivers/iommu/intel/svm.c

125

if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE))

drivers/iommu/intel/svm.c

153

struct intel_iommu *iommu = info->iommu;

drivers/iommu/intel/svm.c

174

ret = __domain_setup_first_level(iommu, dev, pasid,

drivers/iommu/intel/svm.c

28

void intel_svm_check(struct intel_iommu *iommu)

drivers/iommu/intel/svm.c

30

if (!pasid_supported(iommu) || !ecap_smpwc(iommu->ecap))

drivers/iommu/intel/svm.c

34

!cap_fl1gp_support(iommu->cap)) {

drivers/iommu/intel/svm.c

36

iommu->name);

drivers/iommu/intel/svm.c

41

!cap_fl5lp_support(iommu->cap)) {

drivers/iommu/intel/svm.c

43

iommu->name);

drivers/iommu/intel/svm.c

47

iommu->flags |= VTD_FLAG_SVM_CAPABLE;

drivers/iommu/intel/svm.c

92

intel_pasid_tear_down_entry(info->iommu, dev_pasid->dev,

drivers/iommu/intel/trace.h

105

__assign_str(iommu);

drivers/iommu/intel/trace.h

113

__get_str(iommu), __get_str(dev),

drivers/iommu/intel/trace.h

138

__string(iommu, tag->iommu->name)

drivers/iommu/intel/trace.h

149

__assign_str(iommu);

drivers/iommu/intel/trace.h

160

__get_str(iommu), __get_str(dev), __entry->pasid,

drivers/iommu/intel/trace.h

187

#define TRACE_INCLUDE_PATH ../../drivers/iommu/intel/

drivers/iommu/intel/trace.h

22

TP_PROTO(struct intel_iommu *iommu, u64 qw0, u64 qw1, u64 qw2, u64 qw3),

drivers/iommu/intel/trace.h

24

TP_ARGS(iommu, qw0, qw1, qw2, qw3),

drivers/iommu/intel/trace.h

31

__string(iommu, iommu->name)

drivers/iommu/intel/trace.h

35

__assign_str(iommu);

drivers/iommu/intel/trace.h

53

__get_str(iommu),

drivers/iommu/intel/trace.h

59

TP_PROTO(struct intel_iommu *iommu, struct device *dev,

drivers/iommu/intel/trace.h

63

TP_ARGS(iommu, dev, dw0, dw1, dw2, dw3, seq),

drivers/iommu/intel/trace.h

71

__string(iommu, iommu->name)

drivers/iommu/intel/trace.h

82

__assign_str(iommu);

drivers/iommu/intel/trace.h

87

__get_str(iommu), __get_str(dev), __entry->seq,

drivers/iommu/intel/trace.h

97

__string(iommu, tag->iommu->name)

drivers/iommu/io-pgfault.c

23

struct dev_iommu *param = dev->iommu;

drivers/iommu/io-pgfault.c

307

iopf_param = rcu_dereference_check(dev->iommu->fault_param, true);

drivers/iommu/io-pgfault.c

402

struct dev_iommu *param = dev->iommu;

drivers/iommu/io-pgfault.c

469

struct dev_iommu *param = dev->iommu;

drivers/iommu/iommu-priv.h

19

return dev->iommu->iommu_dev->ops;

drivers/iommu/iommu-priv.h

33

int iommu_device_register_bus(struct iommu_device *iommu,

drivers/iommu/iommu-priv.h

37

void iommu_device_unregister_bus(struct iommu_device *iommu,

drivers/iommu/iommu-priv.h

41

int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu);

drivers/iommu/iommu-sva.c

33

if (iommu_mm->pasid >= dev->iommu->max_pasids)

drivers/iommu/iommu-sysfs.c

106

int iommu_device_link(struct iommu_device *iommu, struct device *link)

drivers/iommu/iommu-sysfs.c

110

ret = sysfs_add_link_to_group(&iommu->dev->kobj, "devices",

drivers/iommu/iommu-sysfs.c

115

ret = sysfs_create_link_nowarn(&link->kobj, &iommu->dev->kobj, "iommu");

drivers/iommu/iommu-sysfs.c

117

sysfs_remove_link_from_group(&iommu->dev->kobj, "devices",

drivers/iommu/iommu-sysfs.c

123

void iommu_device_unlink(struct iommu_device *iommu, struct device *link)

drivers/iommu/iommu-sysfs.c

126

sysfs_remove_link_from_group(&iommu->dev->kobj, "devices", dev_name(link));

drivers/iommu/iommu-sysfs.c

54

int iommu_device_sysfs_add(struct iommu_device *iommu,

drivers/iommu/iommu-sysfs.c

62

iommu->dev = kzalloc_obj(*iommu->dev);

drivers/iommu/iommu-sysfs.c

63

if (!iommu->dev)

drivers/iommu/iommu-sysfs.c

66

device_initialize(iommu->dev);

drivers/iommu/iommu-sysfs.c

68

iommu->dev->class = &iommu_class;

drivers/iommu/iommu-sysfs.c

69

iommu->dev->parent = parent;

drivers/iommu/iommu-sysfs.c

70

iommu->dev->groups = groups;

drivers/iommu/iommu-sysfs.c

73

ret = kobject_set_name_vargs(&iommu->dev->kobj, fmt, vargs);

drivers/iommu/iommu-sysfs.c

78

ret = device_add(iommu->dev);

drivers/iommu/iommu-sysfs.c

82

dev_set_drvdata(iommu->dev, iommu);

drivers/iommu/iommu-sysfs.c

87

put_device(iommu->dev);

drivers/iommu/iommu-sysfs.c

92

void iommu_device_sysfs_remove(struct iommu_device *iommu)

drivers/iommu/iommu-sysfs.c

94

dev_set_drvdata(iommu->dev, NULL);

drivers/iommu/iommu-sysfs.c

95

device_unregister(iommu->dev);

drivers/iommu/iommu-sysfs.c

96

iommu->dev = NULL;

drivers/iommu/iommu.c

1221

dev->iommu->require_direct = 1;

drivers/iommu/iommu.c

1605

struct iommu_device *iommu = dev->iommu->iommu_dev;

drivers/iommu/iommu.c

1607

if (!iommu->singleton_group) {

drivers/iommu/iommu.c

1613

iommu->singleton_group = group;

drivers/iommu/iommu.c

1615

return iommu_group_ref_get(iommu->singleton_group);

drivers/iommu/iommu.c

1735

dev->iommu->max_pasids ? IOMMU_HWPT_ALLOC_PASID : 0);

drivers/iommu/iommu.c

2174

dev->iommu->attach_deferred = 0;

drivers/iommu/iommu.c

2226

if (!dev->iommu || !dev->iommu->attach_deferred)

drivers/iommu/iommu.c

2411

if (dev->iommu->require_direct &&

drivers/iommu/iommu.c

2419

if (dev->iommu->attach_deferred) {

drivers/iommu/iommu.c

2422

dev->iommu->attach_deferred = 0;

drivers/iommu/iommu.c

275

if (dev->iommu && dev->iommu->iommu_dev == data)

drivers/iommu/iommu.c

289

int iommu_device_register(struct iommu_device *iommu,

drivers/iommu/iommu.c

298

iommu->ops = ops;

drivers/iommu/iommu.c

300

iommu->fwnode = dev_fwnode(hwdev);

drivers/iommu/iommu.c

303

list_add_tail(&iommu->list, &iommu_device_list);

drivers/iommu/iommu.c

3054

const struct iommu_device *iommu, *ret = NULL;

drivers/iommu/iommu.c

3057

list_for_each_entry(iommu, &iommu_device_list, list)

drivers/iommu/iommu.c

3058

if (iommu->fwnode == fwnode) {

drivers/iommu/iommu.c

3059

ret = iommu;

drivers/iommu/iommu.c

3068

const struct iommu_device *iommu = iommu_from_fwnode(fwnode);

drivers/iommu/iommu.c

3070

return iommu ? iommu->ops : NULL;

drivers/iommu/iommu.c

3075

const struct iommu_device *iommu = iommu_from_fwnode(iommu_fwnode);

drivers/iommu/iommu.c

3078

if (!iommu)

drivers/iommu/iommu.c

3080

if (!dev->iommu && !READ_ONCE(iommu->ready))

drivers/iommu/iommu.c

3084

return iommu->ops == iommu_fwspec_ops(fwspec) ? 0 : -EINVAL;

drivers/iommu/iommu.c

309

iommu_device_unregister(iommu);

drivers/iommu/iommu.c

311

WRITE_ONCE(iommu->ready, true);

drivers/iommu/iommu.c

316

void iommu_device_unregister(struct iommu_device *iommu)

drivers/iommu/iommu.c

3189

gdev->dev->iommu->iommu_dev->dev,

drivers/iommu/iommu.c

319

bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group);

drivers/iommu/iommu.c

322

list_del(&iommu->list);

drivers/iommu/iommu.c

326

iommu_group_put(iommu->singleton_group);

drivers/iommu/iommu.c

327

iommu->singleton_group = NULL;

drivers/iommu/iommu.c

332

void iommu_device_unregister_bus(struct iommu_device *iommu,

drivers/iommu/iommu.c

337

fwnode_remove_software_node(iommu->fwnode);

drivers/iommu/iommu.c

338

iommu_device_unregister(iommu);

drivers/iommu/iommu.c

347

int iommu_device_register_bus(struct iommu_device *iommu,

drivers/iommu/iommu.c

354

iommu->ops = ops;

drivers/iommu/iommu.c

3564

if (device->dev->iommu->max_pasids > 0) {

drivers/iommu/iommu.c

3579

if (device->dev->iommu->max_pasids > 0) {

drivers/iommu/iommu.c

360

iommu->fwnode = fwnode_create_software_node(NULL, NULL);

drivers/iommu/iommu.c

3607

if (!device->blocked && device->dev->iommu->max_pasids > 0)

drivers/iommu/iommu.c

361

if (IS_ERR(iommu->fwnode)) {

drivers/iommu/iommu.c

363

return PTR_ERR(iommu->fwnode);

drivers/iommu/iommu.c

3666

if ((device->dev->iommu->max_pasids > 0) &&

drivers/iommu/iommu.c

3667

(pasid >= device->dev->iommu->max_pasids)) {

drivers/iommu/iommu.c

367

list_add_tail(&iommu->list, &iommu_device_list);

drivers/iommu/iommu.c

372

iommu_device_unregister_bus(iommu, bus, nb);

drivers/iommu/iommu.c

375

WRITE_ONCE(iommu->ready, true);

drivers/iommu/iommu.c

380

int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu)

drivers/iommu/iommu.c

3832

if (!dev->iommu->max_pasids)

drivers/iommu/iommu.c

3840

dev->iommu->max_pasids - 1, GFP_KERNEL);

drivers/iommu/iommu.c

385

rc = iommu_fwspec_init(dev, iommu->fwnode);

drivers/iommu/iommu.c

401

struct dev_iommu *param = dev->iommu;

drivers/iommu/iommu.c

4083

if (pdev->dev.iommu->max_pasids > 0) {

drivers/iommu/iommu.c

413

dev->iommu = param;

drivers/iommu/iommu.c

419

struct dev_iommu *param = dev->iommu;

drivers/iommu/iommu.c

4209

if (pdev->dev.iommu->max_pasids > 0) {

drivers/iommu/iommu.c

421

dev->iommu = NULL;

drivers/iommu/iommu.c

435

return dev->iommu && dev->iommu->iommu_dev;

drivers/iommu/iommu.c

453

return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);

drivers/iommu/iommu.c

461

dev->iommu->priv = priv;

drivers/iommu/iommu.c

485

if (!dev->iommu->fwspec && !dev->driver && dev->bus->dma_configure) {

drivers/iommu/iommu.c

490

if (!dev->iommu || dev->iommu_group)

drivers/iommu/iommu.c

500

ops = iommu_fwspec_ops(dev->iommu->fwspec);

drivers/iommu/iommu.c

516

dev->iommu->iommu_dev = iommu_dev;

drivers/iommu/iommu.c

531

dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);

drivers/iommu/iommu.c

533

dev->iommu->attach_deferred = ops->is_attach_deferred(dev);

drivers/iommu/iommu.c

544

dev->iommu->iommu_dev = NULL;

drivers/iommu/iommu.c

556

iommu_device_unlink(dev->iommu->iommu_dev, dev);

drivers/iommu/iommu.c

575

if (!dev->iommu->attach_deferred && ops->release_domain) {

drivers/iommu/iommu.c

583

if (dev->iommu->require_direct && ops->identity_domain &&

drivers/iommu/iommu.c

786

if (dev->iommu)

drivers/iommu/iommufd/device.c

1640

if (idev->dev->iommu->max_pasids) {

drivers/iommu/iommufd/device.c

1641

cmd->out_max_pasid_log2 = ilog2(idev->dev->iommu->max_pasids);

drivers/iommu/iommufd/selftest.c

118

struct pt_iommu iommu;

drivers/iommu/iommufd/selftest.c

123

PT_IOMMU_CHECK_DOMAIN(struct mock_iommu_domain, iommu, domain);

drivers/iommu/iommufd/selftest.c

124

PT_IOMMU_CHECK_DOMAIN(struct mock_iommu_domain, amdv1.iommu, domain);

drivers/iommu/iommufd/selftest.c

1703

if (!(mock->flags & MOCK_DIRTY_TRACK) || !mock->iommu.ops->set_dirty) {

drivers/iommu/iommufd/selftest.c

1724

mock->iommu.ops->set_dirty(&mock->iommu, iova + i * page_size);

drivers/iommu/iommufd/selftest.c

392

pt_iommu_deinit(&mock->iommu);

drivers/iommu/iommufd/selftest.c

436

mock->amdv1.iommu.nid = NUMA_NO_NODE;

drivers/iommu/ipmmu-vmsa.c

1095

ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, "%s",

drivers/iommu/ipmmu-vmsa.c

1100

ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev);

drivers/iommu/ipmmu-vmsa.c

1102

iommu_device_sysfs_remove(&mmu->iommu);

drivers/iommu/ipmmu-vmsa.c

1111

iommu_device_sysfs_remove(&mmu->iommu);

drivers/iommu/ipmmu-vmsa.c

1112

iommu_device_unregister(&mmu->iommu);

drivers/iommu/ipmmu-vmsa.c

58

struct iommu_device iommu;

drivers/iommu/ipmmu-vmsa.c

846

return &mmu->iommu;

drivers/iommu/msm_iommu.c

120

struct msm_iommu_dev *iommu = NULL;

drivers/iommu/msm_iommu.c

124

list_for_each_entry(iommu, &priv->list_attached, dom_node) {

drivers/iommu/msm_iommu.c

125

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

129

list_for_each_entry(master, &iommu->ctx_list, list)

drivers/iommu/msm_iommu.c

130

SET_CTX_TLBIALL(iommu->base, master->num, 0);

drivers/iommu/msm_iommu.c

132

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

142

struct msm_iommu_dev *iommu = NULL;

drivers/iommu/msm_iommu.c

147

list_for_each_entry(iommu, &priv->list_attached, dom_node) {

drivers/iommu/msm_iommu.c

148

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

152

list_for_each_entry(master, &iommu->ctx_list, list) {

drivers/iommu/msm_iommu.c

156

iova |= GET_CONTEXTIDR_ASID(iommu->base,

drivers/iommu/msm_iommu.c

158

SET_TLBIVA(iommu->base, master->num, iova);

drivers/iommu/msm_iommu.c

163

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

206

static void config_mids(struct msm_iommu_dev *iommu,

drivers/iommu/msm_iommu.c

215

SET_M2VCBR_N(iommu->base, mid, 0);

drivers/iommu/msm_iommu.c

216

SET_CBACR_N(iommu->base, ctx, 0);

drivers/iommu/msm_iommu.c

219

SET_VMID(iommu->base, mid, 0);

drivers/iommu/msm_iommu.c

222

SET_CBNDX(iommu->base, mid, ctx);

drivers/iommu/msm_iommu.c

225

SET_CBVMID(iommu->base, ctx, 0);

drivers/iommu/msm_iommu.c

228

SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx);

drivers/iommu/msm_iommu.c

231

SET_NSCFG(iommu->base, mid, 3);

drivers/iommu/msm_iommu.c

363

struct msm_iommu_dev *iommu, *ret = NULL;

drivers/iommu/msm_iommu.c

366

list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {

drivers/iommu/msm_iommu.c

367

master = list_first_entry(&iommu->ctx_list,

drivers/iommu/msm_iommu.c

371

ret = iommu;

drivers/iommu/msm_iommu.c

381

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

385

iommu = find_iommu_for_dev(dev);

drivers/iommu/msm_iommu.c

388

if (!iommu)

drivers/iommu/msm_iommu.c

391

return &iommu->iommu;

drivers/iommu/msm_iommu.c

399

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

407

list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {

drivers/iommu/msm_iommu.c

408

master = list_first_entry(&iommu->ctx_list,

drivers/iommu/msm_iommu.c

412

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

416

list_for_each_entry(master, &iommu->ctx_list, list) {

drivers/iommu/msm_iommu.c

423

msm_iommu_alloc_ctx(iommu->context_map,

drivers/iommu/msm_iommu.c

424

0, iommu->ncb);

drivers/iommu/msm_iommu.c

429

config_mids(iommu, master);

drivers/iommu/msm_iommu.c

430

__program_context(iommu->base, master->num,

drivers/iommu/msm_iommu.c

433

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

434

list_add(&iommu->dom_node, &priv->list_attached);

drivers/iommu/msm_iommu.c

450

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

461

list_for_each_entry(iommu, &priv->list_attached, dom_node) {

drivers/iommu/msm_iommu.c

462

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

466

list_for_each_entry(master, &iommu->ctx_list, list) {

drivers/iommu/msm_iommu.c

467

msm_iommu_free_ctx(iommu->context_map, master->num);

drivers/iommu/msm_iommu.c

468

__reset_context(iommu->base, master->num);

drivers/iommu/msm_iommu.c

470

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

530

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

539

iommu = list_first_entry(&priv->list_attached,

drivers/iommu/msm_iommu.c

54

static int __enable_clocks(struct msm_iommu_dev *iommu)

drivers/iommu/msm_iommu.c

542

if (list_empty(&iommu->ctx_list))

drivers/iommu/msm_iommu.c

545

master = list_first_entry(&iommu->ctx_list,

drivers/iommu/msm_iommu.c

550

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

555

SET_CTX_TLBIALL(iommu->base, master->num, 0);

drivers/iommu/msm_iommu.c

556

SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA);

drivers/iommu/msm_iommu.c

558

par = GET_PAR(iommu->base, master->num);

drivers/iommu/msm_iommu.c

561

if (GET_NOFAULT_SS(iommu->base, master->num))

drivers/iommu/msm_iommu.c

566

if (GET_FAULT(iommu->base, master->num))

drivers/iommu/msm_iommu.c

569

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

58

ret = clk_enable(iommu->pclk);

drivers/iommu/msm_iommu.c

601

struct msm_iommu_dev **iommu,

drivers/iommu/msm_iommu.c

607

if (list_empty(&(*iommu)->ctx_list)) {

drivers/iommu/msm_iommu.c

614

list_add(&master->list, &(*iommu)->ctx_list);

drivers/iommu/msm_iommu.c

62

if (iommu->clk) {

drivers/iommu/msm_iommu.c

63

ret = clk_enable(iommu->clk);

drivers/iommu/msm_iommu.c

632

struct msm_iommu_dev *iommu = NULL, *iter;

drivers/iommu/msm_iommu.c

639

iommu = iter;

drivers/iommu/msm_iommu.c

644

if (!iommu) {

drivers/iommu/msm_iommu.c

649

ret = insert_iommu_master(dev, &iommu, spec);

drivers/iommu/msm_iommu.c

65

clk_disable(iommu->pclk);

drivers/iommu/msm_iommu.c

658

struct msm_iommu_dev *iommu = dev_id;

drivers/iommu/msm_iommu.c

664

if (!iommu) {

drivers/iommu/msm_iommu.c

670

pr_err("base = %08x\n", (unsigned int)iommu->base);

drivers/iommu/msm_iommu.c

672

ret = __enable_clocks(iommu);

drivers/iommu/msm_iommu.c

676

for (i = 0; i < iommu->ncb; i++) {

drivers/iommu/msm_iommu.c

677

fsr = GET_FSR(iommu->base, i);

drivers/iommu/msm_iommu.c

681

print_ctx_regs(iommu->base, i);

drivers/iommu/msm_iommu.c

682

SET_FSR(iommu->base, i, 0x4000000F);

drivers/iommu/msm_iommu.c

685

__disable_clocks(iommu);

drivers/iommu/msm_iommu.c

71

static void __disable_clocks(struct msm_iommu_dev *iommu)

drivers/iommu/msm_iommu.c

718

struct msm_iommu_dev *iommu;

drivers/iommu/msm_iommu.c

721

iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/msm_iommu.c

722

if (!iommu)

drivers/iommu/msm_iommu.c

725

iommu->dev = &pdev->dev;

drivers/iommu/msm_iommu.c

726

INIT_LIST_HEAD(&iommu->ctx_list);

drivers/iommu/msm_iommu.c

728

iommu->pclk = devm_clk_get_prepared(iommu->dev, "smmu_pclk");

drivers/iommu/msm_iommu.c

729

if (IS_ERR(iommu->pclk))

drivers/iommu/msm_iommu.c

73

if (iommu->clk)

drivers/iommu/msm_iommu.c

730

return dev_err_probe(iommu->dev, PTR_ERR(iommu->pclk),

drivers/iommu/msm_iommu.c

733

iommu->clk = devm_clk_get_prepared(iommu->dev, "iommu_clk");

drivers/iommu/msm_iommu.c

734

if (IS_ERR(iommu->clk))

drivers/iommu/msm_iommu.c

735

return dev_err_probe(iommu->dev, PTR_ERR(iommu->clk),

drivers/iommu/msm_iommu.c

739

iommu->base = devm_ioremap_resource(iommu->dev, r);

drivers/iommu/msm_iommu.c

74

clk_disable(iommu->clk);

drivers/iommu/msm_iommu.c

740

if (IS_ERR(iommu->base)) {

drivers/iommu/msm_iommu.c

741

ret = dev_err_probe(iommu->dev, PTR_ERR(iommu->base), "could not get iommu base\n");

drivers/iommu/msm_iommu.c

746

iommu->irq = platform_get_irq(pdev, 0);

drivers/iommu/msm_iommu.c

747

if (iommu->irq < 0)

drivers/iommu/msm_iommu.c

75

clk_disable(iommu->pclk);

drivers/iommu/msm_iommu.c

750

ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val);

drivers/iommu/msm_iommu.c

752

dev_err(iommu->dev, "could not get ncb\n");

drivers/iommu/msm_iommu.c

755

iommu->ncb = val;

drivers/iommu/msm_iommu.c

757

msm_iommu_reset(iommu->base, iommu->ncb);

drivers/iommu/msm_iommu.c

758

SET_M(iommu->base, 0, 1);

drivers/iommu/msm_iommu.c

759

SET_PAR(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

760

SET_V2PCFG(iommu->base, 0, 1);

drivers/iommu/msm_iommu.c

761

SET_V2PPR(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

762

par = GET_PAR(iommu->base, 0);

drivers/iommu/msm_iommu.c

763

SET_V2PCFG(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

764

SET_M(iommu->base, 0, 0);

drivers/iommu/msm_iommu.c

771

ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL,

drivers/iommu/msm_iommu.c

775

iommu);

drivers/iommu/msm_iommu.c

777

pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret);

drivers/iommu/msm_iommu.c

781

list_add(&iommu->dev_node, &qcom_iommu_devices);

drivers/iommu/msm_iommu.c

783

ret = iommu_device_sysfs_add(&iommu->iommu, iommu->dev, NULL,

drivers/iommu/msm_iommu.c

790

ret = iommu_device_register(&iommu->iommu, &msm_iommu_ops, &pdev->dev);

drivers/iommu/msm_iommu.c

797

iommu->base, iommu->irq, iommu->ncb);

drivers/iommu/msm_iommu.h

60

struct iommu_device iommu;

drivers/iommu/mtk_iommu.c

1429

ret = iommu_device_sysfs_add(&data->iommu, dev, NULL,

drivers/iommu/mtk_iommu.c

1434

ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev);

drivers/iommu/mtk_iommu.c

1446

iommu_device_unregister(&data->iommu);

drivers/iommu/mtk_iommu.c

1448

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu.c

1468

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu.c

1469

iommu_device_unregister(&data->iommu);

drivers/iommu/mtk_iommu.c

264

struct iommu_device iommu;

drivers/iommu/mtk_iommu.c

886

return &data->iommu;

drivers/iommu/mtk_iommu.c

925

return &data->iommu;

drivers/iommu/mtk_iommu_v1.c

114

struct iommu_device iommu;

drivers/iommu/mtk_iommu_v1.c

514

return &data->iommu;

drivers/iommu/mtk_iommu_v1.c

691

ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL,

drivers/iommu/mtk_iommu_v1.c

696

ret = iommu_device_register(&data->iommu, &mtk_iommu_v1_ops, dev);

drivers/iommu/mtk_iommu_v1.c

706

iommu_device_unregister(&data->iommu);

drivers/iommu/mtk_iommu_v1.c

708

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu_v1.c

723

iommu_device_sysfs_remove(&data->iommu);

drivers/iommu/mtk_iommu_v1.c

724

iommu_device_unregister(&data->iommu);

drivers/iommu/of_iommu.c

129

dev_iommu_present = dev->iommu;

drivers/iommu/of_iommu.c

152

else if (err && dev->iommu)

drivers/iommu/omap-iommu.c

1214

err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL,

drivers/iommu/omap-iommu.c

1222

err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);

drivers/iommu/omap-iommu.c

1236

iommu_device_sysfs_remove(&obj->iommu);

drivers/iommu/omap-iommu.c

1245

iommu_device_sysfs_remove(&obj->iommu);

drivers/iommu/omap-iommu.c

1247

iommu_device_unregister(&obj->iommu);

drivers/iommu/omap-iommu.c

1303

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1320

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1321

for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

1322

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1333

iommu--;

drivers/iommu/omap-iommu.c

1334

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1349

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1357

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1358

for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

1359

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1391

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1398

odomain->iommus = kzalloc_objs(*iommu, odomain->num_iommus, GFP_ATOMIC);

drivers/iommu/omap-iommu.c

1402

iommu = odomain->iommus;

drivers/iommu/omap-iommu.c

1403

for (i = 0; i < odomain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

1404

iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC);

drivers/iommu/omap-iommu.c

1405

if (!iommu->pgtable)

drivers/iommu/omap-iommu.c

1412

if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable,

drivers/iommu/omap-iommu.c

1423

struct omap_iommu_device *iommu = odomain->iommus;

drivers/iommu/omap-iommu.c

1425

for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++)

drivers/iommu/omap-iommu.c

1426

kfree(iommu->pgtable);

drivers/iommu/omap-iommu.c

1438

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

1464

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1465

for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) {

drivers/iommu/omap-iommu.c

1468

ret = omap_iommu_attach(oiommu, iommu->pgtable);

drivers/iommu/omap-iommu.c

1475

iommu->iommu_dev = oiommu;

drivers/iommu/omap-iommu.c

1484

iommu--;

drivers/iommu/omap-iommu.c

1486

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1488

iommu->iommu_dev = NULL;

drivers/iommu/omap-iommu.c

1502

struct omap_iommu_device *iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1521

iommu += (omap_domain->num_iommus - 1);

drivers/iommu/omap-iommu.c

1523

for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) {

drivers/iommu/omap-iommu.c

1524

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1528

iommu->iommu_dev = NULL;

drivers/iommu/omap-iommu.c

1599

struct omap_iommu_device *iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

1600

struct omap_iommu *oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

1695

return &oiommu->iommu;

drivers/iommu/omap-iommu.c

952

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

959

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

960

iommu += (omap_domain->num_iommus - 1);

drivers/iommu/omap-iommu.c

961

for (i = 0; i < omap_domain->num_iommus; i++, iommu--) {

drivers/iommu/omap-iommu.c

962

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.c

981

struct omap_iommu_device *iommu;

drivers/iommu/omap-iommu.c

988

iommu = omap_domain->iommus;

drivers/iommu/omap-iommu.c

989

for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {

drivers/iommu/omap-iommu.c

990

oiommu = iommu->iommu_dev;

drivers/iommu/omap-iommu.h

82

struct iommu_device iommu;

drivers/iommu/riscv/iommu-pci.c

101

riscv_iommu_remove(iommu);

drivers/iommu/riscv/iommu-pci.c

106

struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);

drivers/iommu/riscv/iommu-pci.c

108

riscv_iommu_disable(iommu);

drivers/iommu/riscv/iommu-pci.c

36

struct riscv_iommu_device *iommu;

drivers/iommu/riscv/iommu-pci.c

53

iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/riscv/iommu-pci.c

54

if (!iommu)

drivers/iommu/riscv/iommu-pci.c

57

iommu->dev = dev;

drivers/iommu/riscv/iommu-pci.c

58

iommu->reg = pcim_iomap_table(pdev)[0];

drivers/iommu/riscv/iommu-pci.c

61

dev_set_drvdata(dev, iommu);

drivers/iommu/riscv/iommu-pci.c

64

iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);

drivers/iommu/riscv/iommu-pci.c

65

iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);

drivers/iommu/riscv/iommu-pci.c

68

switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {

drivers/iommu/riscv/iommu-pci.c

84

iommu->irqs_count = rc;

drivers/iommu/riscv/iommu-pci.c

85

for (vec = 0; vec < iommu->irqs_count; vec++)

drivers/iommu/riscv/iommu-pci.c

86

iommu->irqs[vec] = msi_get_virq(dev, vec);

drivers/iommu/riscv/iommu-pci.c

89

if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {

drivers/iommu/riscv/iommu-pci.c

90

iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;

drivers/iommu/riscv/iommu-pci.c

91

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);

drivers/iommu/riscv/iommu-pci.c

94

return riscv_iommu_init(iommu);

drivers/iommu/riscv/iommu-pci.c

99

struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);

drivers/iommu/riscv/iommu-platform.c

101

if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {

drivers/iommu/riscv/iommu-platform.c

102

iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;

drivers/iommu/riscv/iommu-platform.c

103

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);

drivers/iommu/riscv/iommu-platform.c

123

iommu->irqs_count = ret;

drivers/iommu/riscv/iommu-platform.c

125

if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT)

drivers/iommu/riscv/iommu-platform.c

126

iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;

drivers/iommu/riscv/iommu-platform.c

128

for (vec = 0; vec < iommu->irqs_count; vec++)

drivers/iommu/riscv/iommu-platform.c

129

iommu->irqs[vec] = platform_get_irq(pdev, vec);

drivers/iommu/riscv/iommu-platform.c

132

if (!(iommu->fctl & RISCV_IOMMU_FCTL_WSI)) {

drivers/iommu/riscv/iommu-platform.c

133

iommu->fctl |= RISCV_IOMMU_FCTL_WSI;

drivers/iommu/riscv/iommu-platform.c

134

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);

drivers/iommu/riscv/iommu-platform.c

142

return riscv_iommu_init(iommu);

drivers/iommu/riscv/iommu-platform.c

147

struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);

drivers/iommu/riscv/iommu-platform.c

148

bool msi = !(iommu->fctl & RISCV_IOMMU_FCTL_WSI);

drivers/iommu/riscv/iommu-platform.c

150

riscv_iommu_remove(iommu);

drivers/iommu/riscv/iommu-platform.c

27

struct riscv_iommu_device *iommu = dev_get_drvdata(dev);

drivers/iommu/riscv/iommu-platform.c

41

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_ADDR(idx), addr);

drivers/iommu/riscv/iommu-platform.c

42

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_DATA(idx), msg->data);

drivers/iommu/riscv/iommu-platform.c

43

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_CTRL(idx), 0);

drivers/iommu/riscv/iommu-platform.c

50

struct riscv_iommu_device *iommu = NULL;

drivers/iommu/riscv/iommu-platform.c

55

iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/riscv/iommu-platform.c

56

if (!iommu)

drivers/iommu/riscv/iommu-platform.c

59

iommu->dev = dev;

drivers/iommu/riscv/iommu-platform.c

60

iommu->reg = devm_platform_get_and_ioremap_resource(pdev, 0, &res);

drivers/iommu/riscv/iommu-platform.c

61

if (IS_ERR(iommu->reg))

drivers/iommu/riscv/iommu-platform.c

62

return dev_err_probe(dev, PTR_ERR(iommu->reg),

drivers/iommu/riscv/iommu-platform.c

65

dev_set_drvdata(dev, iommu);

drivers/iommu/riscv/iommu-platform.c

68

iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);

drivers/iommu/riscv/iommu-platform.c

69

iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);

drivers/iommu/riscv/iommu-platform.c

71

iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;

drivers/iommu/riscv/iommu-platform.c

73

igs = FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps);

drivers/iommu/riscv/iommu-platform.c

90

ret = platform_device_msi_init_and_alloc_irqs(dev, iommu->irqs_count,

drivers/iommu/riscv/iommu-platform.c

97

for (vec = 0; vec < iommu->irqs_count; vec++)

drivers/iommu/riscv/iommu-platform.c

98

iommu->irqs[vec] = msi_get_virq(dev, vec);

drivers/iommu/riscv/iommu.c

1002

if (iommu->caps & RISCV_IOMMU_CAPABILITIES_S &&

drivers/iommu/riscv/iommu.c

1008

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

101

devres_release(iommu->dev, riscv_iommu_devres_pages_release,

drivers/iommu/riscv/iommu.c

1020

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

1026

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

1032

struct riscv_iommu_device *iommu, *prev;

drivers/iommu/riscv/iommu.c

1066

iommu = dev_to_iommu(bond->dev);

drivers/iommu/riscv/iommu.c

1074

if (iommu == prev)

drivers/iommu/riscv/iommu.c

1077

riscv_iommu_iotlb_inval_iommu(iommu, domain->pscid, &tlbi);

drivers/iommu/riscv/iommu.c

1078

prev = iommu;

drivers/iommu/riscv/iommu.c

1083

iommu = dev_to_iommu(bond->dev);

drivers/iommu/riscv/iommu.c

1084

if (iommu == prev)

drivers/iommu/riscv/iommu.c

1087

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

1088

prev = iommu;

drivers/iommu/riscv/iommu.c

1099

static void riscv_iommu_iodir_iotinval(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

1153

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

1167

static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

1178

dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);

drivers/iommu/riscv/iommu.c

1188

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

1193

riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL);

drivers/iommu/riscv/iommu.c

1198

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

1205

dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);

drivers/iommu/riscv/iommu.c

1218

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

1223

riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL);

drivers/iommu/riscv/iommu.c

1226

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

1264

pt_iommu_deinit(&domain->riscvpt.iommu);

drivers/iommu/riscv/iommu.c

1268

static bool riscv_iommu_pt_supported(struct riscv_iommu_device *iommu, int pgd_mode)

drivers/iommu/riscv/iommu.c

1272

return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39;

drivers/iommu/riscv/iommu.c

1275

return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48;

drivers/iommu/riscv/iommu.c

1278

return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57;

drivers/iommu/riscv/iommu.c

1288

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1295

if (!riscv_iommu_pt_supported(iommu, pt_info.fsc_iosatp_mode))

drivers/iommu/riscv/iommu.c

1306

riscv_iommu_iodir_update(iommu, dev, fsc, ta);

drivers/iommu/riscv/iommu.c

132

static int riscv_iommu_queue_alloc(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

1325

struct riscv_iommu_device *iommu;

drivers/iommu/riscv/iommu.c

1328

iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1329

if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57) {

drivers/iommu/riscv/iommu.c

1331

} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48) {

drivers/iommu/riscv/iommu.c

1333

} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39) {

drivers/iommu/riscv/iommu.c

1355

if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SVPBMT)

drivers/iommu/riscv/iommu.c

1357

domain->riscvpt.iommu.nid = dev_to_node(iommu->dev);

drivers/iommu/riscv/iommu.c

1379

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1383

riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, 0);

drivers/iommu/riscv/iommu.c

1401

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

1404

riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, RISCV_IOMMU_PC_TA_V);

drivers/iommu/riscv/iommu.c

143

riscv_iommu_writeq(iommu, queue->qbr, RISCV_IOMMU_QUEUE_LOG2SZ_FIELD);

drivers/iommu/riscv/iommu.c

1433

struct riscv_iommu_device *iommu;

drivers/iommu/riscv/iommu.c

144

qb = riscv_iommu_readq(iommu, queue->qbr);

drivers/iommu/riscv/iommu.c

1442

iommu = dev_get_drvdata(fwspec->iommu_fwnode->dev);

drivers/iommu/riscv/iommu.c

1443

if (!iommu)

drivers/iommu/riscv/iommu.c

1450

if (iommu->ddt_mode <= RISCV_IOMMU_DDTP_IOMMU_MODE_BARE)

drivers/iommu/riscv/iommu.c

1462

dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);

drivers/iommu/riscv/iommu.c

1474

return &iommu->iommu;

drivers/iommu/riscv/iommu.c

1495

static int riscv_iommu_init_check(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

1504

ddtp = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_DDTP);

drivers/iommu/riscv/iommu.c

1512

riscv_iommu_disable(iommu);

drivers/iommu/riscv/iommu.c

1517

!!(iommu->fctl & RISCV_IOMMU_FCTL_BE)) {

drivers/iommu/riscv/iommu.c

1518

if (!(iommu->caps & RISCV_IOMMU_CAPABILITIES_END))

drivers/iommu/riscv/iommu.c

1520

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL,

drivers/iommu/riscv/iommu.c

1521

iommu->fctl ^ RISCV_IOMMU_FCTL_BE);

drivers/iommu/riscv/iommu.c

1522

iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);

drivers/iommu/riscv/iommu.c

1524

!!(iommu->fctl & RISCV_IOMMU_FCTL_BE))

drivers/iommu/riscv/iommu.c

1532

if (!iommu->irqs_count)

drivers/iommu/riscv/iommu.c

1535

iommu->icvec = FIELD_PREP(RISCV_IOMMU_ICVEC_FIV, 1 % iommu->irqs_count) |

drivers/iommu/riscv/iommu.c

1536

FIELD_PREP(RISCV_IOMMU_ICVEC_PIV, 2 % iommu->irqs_count) |

drivers/iommu/riscv/iommu.c

1537

FIELD_PREP(RISCV_IOMMU_ICVEC_PMIV, 3 % iommu->irqs_count);

drivers/iommu/riscv/iommu.c

1538

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_ICVEC, iommu->icvec);

drivers/iommu/riscv/iommu.c

1539

iommu->icvec = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_ICVEC);

drivers/iommu/riscv/iommu.c

1540

if (max3(FIELD_GET(RISCV_IOMMU_ICVEC_CIV, iommu->icvec),

drivers/iommu/riscv/iommu.c

1541

FIELD_GET(RISCV_IOMMU_ICVEC_FIV, iommu->icvec),

drivers/iommu/riscv/iommu.c

1542

max(FIELD_GET(RISCV_IOMMU_ICVEC_PIV, iommu->icvec),

drivers/iommu/riscv/iommu.c

1543

FIELD_GET(RISCV_IOMMU_ICVEC_PMIV, iommu->icvec))) >= iommu->irqs_count)

drivers/iommu/riscv/iommu.c

1549

void riscv_iommu_remove(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

1551

iommu_device_unregister(&iommu->iommu);

drivers/iommu/riscv/iommu.c

1552

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/riscv/iommu.c

1553

riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);

drivers/iommu/riscv/iommu.c

1554

riscv_iommu_queue_disable(&iommu->cmdq);

drivers/iommu/riscv/iommu.c

1555

riscv_iommu_queue_disable(&iommu->fltq);

drivers/iommu/riscv/iommu.c

1558

int riscv_iommu_init(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

1562

RISCV_IOMMU_QUEUE_INIT(&iommu->cmdq, CQ);

drivers/iommu/riscv/iommu.c

1563

RISCV_IOMMU_QUEUE_INIT(&iommu->fltq, FQ);

drivers/iommu/riscv/iommu.c

1565

rc = riscv_iommu_init_check(iommu);

drivers/iommu/riscv/iommu.c

1567

return dev_err_probe(iommu->dev, rc, "unexpected device state\n");

drivers/iommu/riscv/iommu.c

1569

rc = riscv_iommu_iodir_alloc(iommu);

drivers/iommu/riscv/iommu.c

1573

rc = riscv_iommu_queue_alloc(iommu, &iommu->cmdq,

drivers/iommu/riscv/iommu.c

1578

rc = riscv_iommu_queue_alloc(iommu, &iommu->fltq,

drivers/iommu/riscv/iommu.c

1583

rc = riscv_iommu_queue_enable(iommu, &iommu->cmdq, riscv_iommu_cmdq_process);

drivers/iommu/riscv/iommu.c

1587

rc = riscv_iommu_queue_enable(iommu, &iommu->fltq, riscv_iommu_fltq_process);

drivers/iommu/riscv/iommu.c

1591

rc = riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_MAX);

drivers/iommu/riscv/iommu.c

1595

rc = iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "riscv-iommu@%s",

drivers/iommu/riscv/iommu.c

1596

dev_name(iommu->dev));

drivers/iommu/riscv/iommu.c

1598

dev_err_probe(iommu->dev, rc, "cannot register sysfs interface\n");

drivers/iommu/riscv/iommu.c

1603

rc = rimt_iommu_register(iommu->dev);

drivers/iommu/riscv/iommu.c

1605

dev_err_probe(iommu->dev, rc, "cannot register iommu with RIMT\n");

drivers/iommu/riscv/iommu.c

1610

rc = iommu_device_register(&iommu->iommu, &riscv_iommu_ops, iommu->dev);

drivers/iommu/riscv/iommu.c

1612

dev_err_probe(iommu->dev, rc, "cannot register iommu interface\n");

drivers/iommu/riscv/iommu.c

1619

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/riscv/iommu.c

1621

riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);

drivers/iommu/riscv/iommu.c

1623

riscv_iommu_queue_disable(&iommu->fltq);

drivers/iommu/riscv/iommu.c

1624

riscv_iommu_queue_disable(&iommu->cmdq);

drivers/iommu/riscv/iommu.c

164

queue->base = devm_ioremap(iommu->dev, queue->phys, queue_size);

drivers/iommu/riscv/iommu.c

170

iommu, max(queue_size, SZ_4K));

drivers/iommu/riscv/iommu.c

182

riscv_iommu_writeq(iommu, queue->qbr, qb);

drivers/iommu/riscv/iommu.c

183

rb = riscv_iommu_readq(iommu, queue->qbr);

drivers/iommu/riscv/iommu.c

185

dev_err(iommu->dev, "queue #%u allocation failed\n", queue->qid);

drivers/iommu/riscv/iommu.c

192

dev_dbg(iommu->dev, "queue #%u allocated 2^%u entries",

drivers/iommu/riscv/iommu.c

203

if (riscv_iommu_readl(queue->iommu, RISCV_IOMMU_REG_IPSR) & Q_IPSR(queue))

drivers/iommu/riscv/iommu.c

209

static int riscv_iommu_queue_vec(struct riscv_iommu_device *iommu, int n)

drivers/iommu/riscv/iommu.c

212

return (iommu->icvec >> (n * 4)) & RISCV_IOMMU_ICVEC_CIV;

drivers/iommu/riscv/iommu.c

221

static int riscv_iommu_queue_enable(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

225

const unsigned int irq = iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)];

drivers/iommu/riscv/iommu.c

229

if (queue->iommu)

drivers/iommu/riscv/iommu.c

236

queue->iommu = iommu;

drivers/iommu/riscv/iommu.c

239

dev_name(iommu->dev), queue);

drivers/iommu/riscv/iommu.c

241

queue->iommu = NULL;

drivers/iommu/riscv/iommu.c

247

riscv_iommu_writel(queue->iommu, Q_TAIL(queue), 0);

drivers/iommu/riscv/iommu.c

249

riscv_iommu_writel(queue->iommu, Q_HEAD(queue), 0);

drivers/iommu/riscv/iommu.c

257

riscv_iommu_writel(iommu, queue->qcr,

drivers/iommu/riscv/iommu.c

262

riscv_iommu_readl_timeout(iommu, queue->qcr,

drivers/iommu/riscv/iommu.c

270

riscv_iommu_writel(iommu, queue->qcr, 0);

drivers/iommu/riscv/iommu.c

272

queue->iommu = NULL;

drivers/iommu/riscv/iommu.c

273

dev_err(iommu->dev, "queue #%u failed to start\n", queue->qid);

drivers/iommu/riscv/iommu.c

278

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));

drivers/iommu/riscv/iommu.c

289

struct riscv_iommu_device *iommu = queue->iommu;

drivers/iommu/riscv/iommu.c

292

if (!iommu)

drivers/iommu/riscv/iommu.c

295

free_irq(iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)], queue);

drivers/iommu/riscv/iommu.c

296

riscv_iommu_writel(iommu, queue->qcr, 0);

drivers/iommu/riscv/iommu.c

297

riscv_iommu_readl_timeout(iommu, queue->qcr,

drivers/iommu/riscv/iommu.c

302

dev_err(iommu->dev, "fail to disable hardware queue #%u, csr 0x%x\n",

drivers/iommu/riscv/iommu.c

305

queue->iommu = NULL;

drivers/iommu/riscv/iommu.c

326

if (riscv_iommu_readl_timeout(queue->iommu, Q_TAIL(queue),

drivers/iommu/riscv/iommu.c

329

dev_err_once(queue->iommu->dev,

drivers/iommu/riscv/iommu.c

348

riscv_iommu_writel(queue->iommu, Q_HEAD(queue), Q_ITEM(queue, head));

drivers/iommu/riscv/iommu.c

358

if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,

drivers/iommu/riscv/iommu.c

381

(riscv_iommu_readl(queue->iommu, queue->qcr) & flags) ||

drivers/iommu/riscv/iommu.c

415

if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,

drivers/iommu/riscv/iommu.c

436

riscv_iommu_writel(queue->iommu, Q_TAIL(queue), Q_ITEM(queue, prod + 1));

drivers/iommu/riscv/iommu.c

45

iommu_get_iommu_dev(dev, struct riscv_iommu_device, iommu)

drivers/iommu/riscv/iommu.c

454

dev_err_once(queue->iommu->dev, "Hardware error: command enqueue failed\n");

drivers/iommu/riscv/iommu.c

470

ctrl = riscv_iommu_readl(queue->iommu, queue->qcr);

drivers/iommu/riscv/iommu.c

473

riscv_iommu_writel(queue->iommu, queue->qcr, ctrl);

drivers/iommu/riscv/iommu.c

474

dev_warn(queue->iommu->dev,

drivers/iommu/riscv/iommu.c

486

riscv_iommu_writel(queue->iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));

drivers/iommu/riscv/iommu.c

492

static void riscv_iommu_cmd_send(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

495

riscv_iommu_queue_send(&iommu->cmdq, cmd, sizeof(*cmd));

drivers/iommu/riscv/iommu.c

499

static void riscv_iommu_cmd_sync(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

506

prod = riscv_iommu_queue_send(&iommu->cmdq, &cmd, sizeof(cmd));

drivers/iommu/riscv/iommu.c

511

if (riscv_iommu_queue_wait(&iommu->cmdq, prod, timeout_us))

drivers/iommu/riscv/iommu.c

512

dev_err_once(iommu->dev,

drivers/iommu/riscv/iommu.c

520

static void riscv_iommu_fault(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

528

dev_warn_ratelimited(iommu->dev,

drivers/iommu/riscv/iommu.c

537

struct riscv_iommu_device *iommu = queue->iommu;

drivers/iommu/riscv/iommu.c

545

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));

drivers/iommu/riscv/iommu.c

550

riscv_iommu_fault(iommu, &events[Q_ITEM(queue, idx)]);

drivers/iommu/riscv/iommu.c

555

ctrl = riscv_iommu_readl(iommu, queue->qcr);

drivers/iommu/riscv/iommu.c

557

riscv_iommu_writel(iommu, queue->qcr, ctrl);

drivers/iommu/riscv/iommu.c

558

dev_warn(iommu->dev,

drivers/iommu/riscv/iommu.c

569

static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

572

const bool base_format = !(iommu->caps & RISCV_IOMMU_CAPABILITIES_MSI_FLAT);

drivers/iommu/riscv/iommu.c

580

if (iommu->ddt_mode < RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL ||

drivers/iommu/riscv/iommu.c

581

iommu->ddt_mode > RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL)

drivers/iommu/riscv/iommu.c

606

depth = iommu->ddt_mode - RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL;

drivers/iommu/riscv/iommu.c

611

for (ddtp = iommu->ddt_root; depth-- > 0;) {

drivers/iommu/riscv/iommu.c

630

ptr = riscv_iommu_get_pages(iommu, SZ_4K);

drivers/iommu/riscv/iommu.c

643

riscv_iommu_free_pages(iommu, ptr);

drivers/iommu/riscv/iommu.c

661

void riscv_iommu_disable(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

663

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,

drivers/iommu/riscv/iommu.c

666

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_CQCSR, 0);

drivers/iommu/riscv/iommu.c

667

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FQCSR, 0);

drivers/iommu/riscv/iommu.c

668

riscv_iommu_writel(iommu, RISCV_IOMMU_REG_PQCSR, 0);

drivers/iommu/riscv/iommu.c

671

#define riscv_iommu_read_ddtp(iommu) ({ \

drivers/iommu/riscv/iommu.c

673

riscv_iommu_readq_timeout((iommu), RISCV_IOMMU_REG_DDTP, ddtp, \

drivers/iommu/riscv/iommu.c

678

static int riscv_iommu_iodir_alloc(struct riscv_iommu_device *iommu)

drivers/iommu/riscv/iommu.c

683

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

695

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,

drivers/iommu/riscv/iommu.c

697

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

701

iommu->ddt_phys = ppn_to_phys(ddtp);

drivers/iommu/riscv/iommu.c

702

if (iommu->ddt_phys)

drivers/iommu/riscv/iommu.c

703

iommu->ddt_root = devm_ioremap(iommu->dev,

drivers/iommu/riscv/iommu.c

704

iommu->ddt_phys, PAGE_SIZE);

drivers/iommu/riscv/iommu.c

705

if (iommu->ddt_root)

drivers/iommu/riscv/iommu.c

706

memset(iommu->ddt_root, 0, PAGE_SIZE);

drivers/iommu/riscv/iommu.c

709

if (!iommu->ddt_root) {

drivers/iommu/riscv/iommu.c

71

static void *riscv_iommu_get_pages(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

710

iommu->ddt_root = riscv_iommu_get_pages(iommu, SZ_4K);

drivers/iommu/riscv/iommu.c

711

iommu->ddt_phys = __pa(iommu->ddt_root);

drivers/iommu/riscv/iommu.c

714

if (!iommu->ddt_root)

drivers/iommu/riscv/iommu.c

725

static int riscv_iommu_iodir_set_mode(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

728

struct device *dev = iommu->dev;

drivers/iommu/riscv/iommu.c

733

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

748

rq_ddtp |= phys_to_ppn(iommu->ddt_phys);

drivers/iommu/riscv/iommu.c

750

riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP, rq_ddtp);

drivers/iommu/riscv/iommu.c

751

ddtp = riscv_iommu_read_ddtp(iommu);

drivers/iommu/riscv/iommu.c

77

addr = iommu_alloc_pages_node_sz(dev_to_node(iommu->dev),

drivers/iommu/riscv/iommu.c

795

iommu->ddt_mode = mode;

drivers/iommu/riscv/iommu.c

801

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

805

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

808

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

823

PT_IOMMU_CHECK_DOMAIN(struct riscv_iommu_domain, riscvpt.iommu, domain);

drivers/iommu/riscv/iommu.c

858

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

874

if (dev_to_iommu(list_entry(bonds, struct riscv_iommu_bond, list)->dev) == iommu)

drivers/iommu/riscv/iommu.c

888

struct riscv_iommu_device *iommu = dev_to_iommu(dev);

drivers/iommu/riscv/iommu.c

902

else if (dev_to_iommu(bond->dev) == iommu)

drivers/iommu/riscv/iommu.c

917

riscv_iommu_cmd_send(iommu, &cmd);

drivers/iommu/riscv/iommu.c

919

riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);

drivers/iommu/riscv/iommu.c

92

devres_add(iommu->dev, devres);

drivers/iommu/riscv/iommu.c

97

static void riscv_iommu_free_pages(struct riscv_iommu_device *iommu, void *addr)

drivers/iommu/riscv/iommu.c

983

static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu,

drivers/iommu/riscv/iommu.c

988

(iommu->caps & RISCV_IOMMU_CAPABILITIES_NL);

drivers/iommu/riscv/iommu.h

28

struct riscv_iommu_device *iommu; /* iommu device handling the queue when active */

drivers/iommu/riscv/iommu.h

38

struct iommu_device iommu;

drivers/iommu/riscv/iommu.h

65

int riscv_iommu_init(struct riscv_iommu_device *iommu);

drivers/iommu/riscv/iommu.h

66

void riscv_iommu_remove(struct riscv_iommu_device *iommu);

drivers/iommu/riscv/iommu.h

67

void riscv_iommu_disable(struct riscv_iommu_device *iommu);

drivers/iommu/riscv/iommu.h

69

#define riscv_iommu_readl(iommu, addr) \

drivers/iommu/riscv/iommu.h

70

readl_relaxed((iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

72

#define riscv_iommu_readq(iommu, addr) \

drivers/iommu/riscv/iommu.h

73

readq_relaxed((iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

75

#define riscv_iommu_writel(iommu, addr, val) \

drivers/iommu/riscv/iommu.h

76

writel_relaxed((val), (iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

78

#define riscv_iommu_writeq(iommu, addr, val) \

drivers/iommu/riscv/iommu.h

79

writeq_relaxed((val), (iommu)->reg + (addr))

drivers/iommu/riscv/iommu.h

81

#define riscv_iommu_readq_timeout(iommu, addr, val, cond, delay_us, timeout_us) \

drivers/iommu/riscv/iommu.h

82

readx_poll_timeout(readq_relaxed, (iommu)->reg + (addr), val, cond, \

drivers/iommu/riscv/iommu.h

85

#define riscv_iommu_readl_timeout(iommu, addr, val, cond, delay_us, timeout_us) \

drivers/iommu/riscv/iommu.h

86

readx_poll_timeout(readl_relaxed, (iommu)->reg + (addr), val, cond, \

drivers/iommu/rockchip-iommu.c

1000

rk_iommu_disable(iommu);

drivers/iommu/rockchip-iommu.c

1001

pm_runtime_put(iommu->dev);

drivers/iommu/rockchip-iommu.c

1019

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1028

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

1029

if (!iommu)

drivers/iommu/rockchip-iommu.c

1035

if (iommu->domain == domain)

drivers/iommu/rockchip-iommu.c

1042

iommu->domain = domain;

drivers/iommu/rockchip-iommu.c

1045

list_add_tail(&iommu->node, &rk_domain->iommus);

drivers/iommu/rockchip-iommu.c

1048

ret = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/rockchip-iommu.c

1052

ret = rk_iommu_enable(iommu);

drivers/iommu/rockchip-iommu.c

1062

iommu->domain));

drivers/iommu/rockchip-iommu.c

1065

pm_runtime_put(iommu->dev);

drivers/iommu/rockchip-iommu.c

1073

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1089

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

1090

rk_domain->dma_dev = iommu->dev;

drivers/iommu/rockchip-iommu.c

1146

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1152

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

1154

data->link = device_link_add(dev, iommu->dev,

drivers/iommu/rockchip-iommu.c

1157

return &iommu->iommu;

drivers/iommu/rockchip-iommu.c

1179

data->iommu = platform_get_drvdata(iommu_dev);

drivers/iommu/rockchip-iommu.c

119

struct iommu_device iommu;

drivers/iommu/rockchip-iommu.c

1206

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1212

iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/rockchip-iommu.c

1213

if (!iommu)

drivers/iommu/rockchip-iommu.c

1216

iommu->domain = &rk_identity_domain;

drivers/iommu/rockchip-iommu.c

1218

platform_set_drvdata(pdev, iommu);

drivers/iommu/rockchip-iommu.c

1219

iommu->dev = dev;

drivers/iommu/rockchip-iommu.c

1220

iommu->num_mmu = 0;

drivers/iommu/rockchip-iommu.c

1233

iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases),

drivers/iommu/rockchip-iommu.c

1235

if (!iommu->bases)

drivers/iommu/rockchip-iommu.c

1242

iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res);

drivers/iommu/rockchip-iommu.c

1243

if (IS_ERR(iommu->bases[i]))

drivers/iommu/rockchip-iommu.c

1245

iommu->num_mmu++;

drivers/iommu/rockchip-iommu.c

1247

if (iommu->num_mmu == 0)

drivers/iommu/rockchip-iommu.c

1248

return PTR_ERR(iommu->bases[0]);

drivers/iommu/rockchip-iommu.c

1250

iommu->num_irq = platform_irq_count(pdev);

drivers/iommu/rockchip-iommu.c

1251

if (iommu->num_irq < 0)

drivers/iommu/rockchip-iommu.c

1252

return iommu->num_irq;

drivers/iommu/rockchip-iommu.c

1254

iommu->reset_disabled = device_property_read_bool(dev,

drivers/iommu/rockchip-iommu.c

1257

iommu->num_clocks = ARRAY_SIZE(rk_iommu_clocks);

drivers/iommu/rockchip-iommu.c

1258

iommu->clocks = devm_kcalloc(iommu->dev, iommu->num_clocks,

drivers/iommu/rockchip-iommu.c

1259

sizeof(*iommu->clocks), GFP_KERNEL);

drivers/iommu/rockchip-iommu.c

126

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

1260

if (!iommu->clocks)

drivers/iommu/rockchip-iommu.c

1263

for (i = 0; i < iommu->num_clocks; ++i)

drivers/iommu/rockchip-iommu.c

1264

iommu->clocks[i].id = rk_iommu_clocks[i];

drivers/iommu/rockchip-iommu.c

1271

err = devm_clk_bulk_get(iommu->dev, iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

1273

iommu->num_clocks = 0;

drivers/iommu/rockchip-iommu.c

1277

err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

1283

for (i = 0; i < iommu->num_irq; i++) {

drivers/iommu/rockchip-iommu.c

1291

err = devm_request_irq(iommu->dev, irq, rk_iommu_irq,

drivers/iommu/rockchip-iommu.c

1292

IRQF_SHARED, dev_name(dev), iommu);

drivers/iommu/rockchip-iommu.c

1299

err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));

drivers/iommu/rockchip-iommu.c

1303

err = iommu_device_register(&iommu->iommu, &rk_iommu_ops, dev);

drivers/iommu/rockchip-iommu.c

1309

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/rockchip-iommu.c

1312

clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

1318

struct rk_iommu *iommu = platform_get_drvdata(pdev);

drivers/iommu/rockchip-iommu.c

1321

for (i = 0; i < iommu->num_irq; i++) {

drivers/iommu/rockchip-iommu.c

1324

devm_free_irq(iommu->dev, irq, iommu);

drivers/iommu/rockchip-iommu.c

1332

struct rk_iommu *iommu = dev_get_drvdata(dev);

drivers/iommu/rockchip-iommu.c

1334

if (iommu->domain == &rk_identity_domain)

drivers/iommu/rockchip-iommu.c

1337

rk_iommu_disable(iommu);

drivers/iommu/rockchip-iommu.c

1343

struct rk_iommu *iommu = dev_get_drvdata(dev);

drivers/iommu/rockchip-iommu.c

1345

if (iommu->domain == &rk_identity_domain)

drivers/iommu/rockchip-iommu.c

1348

return rk_iommu_enable(iommu);

drivers/iommu/rockchip-iommu.c

349

static void rk_iommu_command(struct rk_iommu *iommu, u32 command)

drivers/iommu/rockchip-iommu.c

353

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

354

writel(command, iommu->bases[i] + RK_MMU_COMMAND);

drivers/iommu/rockchip-iommu.c

361

static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova_start,

drivers/iommu/rockchip-iommu.c

370

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

374

rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova);

drivers/iommu/rockchip-iommu.c

378

static bool rk_iommu_is_stall_active(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

383

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

384

active &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &

drivers/iommu/rockchip-iommu.c

390

static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

395

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

396

enable &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &

drivers/iommu/rockchip-iommu.c

402

static bool rk_iommu_is_reset_done(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

407

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

408

done &= rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0;

drivers/iommu/rockchip-iommu.c

413

static int rk_iommu_enable_stall(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

418

if (rk_iommu_is_stall_active(iommu))

drivers/iommu/rockchip-iommu.c

422

if (!rk_iommu_is_paging_enabled(iommu))

drivers/iommu/rockchip-iommu.c

425

rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL);

drivers/iommu/rockchip-iommu.c

427

ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val,

drivers/iommu/rockchip-iommu.c

431

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

432

dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

433

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

438

static int rk_iommu_disable_stall(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

443

if (!rk_iommu_is_stall_active(iommu))

drivers/iommu/rockchip-iommu.c

446

rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL);

drivers/iommu/rockchip-iommu.c

448

ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val,

drivers/iommu/rockchip-iommu.c

452

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

453

dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

454

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

459

static int rk_iommu_enable_paging(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

464

if (rk_iommu_is_paging_enabled(iommu))

drivers/iommu/rockchip-iommu.c

467

rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING);

drivers/iommu/rockchip-iommu.c

469

ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val,

drivers/iommu/rockchip-iommu.c

473

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

474

dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

475

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

480

static int rk_iommu_disable_paging(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

485

if (!rk_iommu_is_paging_enabled(iommu))

drivers/iommu/rockchip-iommu.c

488

rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING);

drivers/iommu/rockchip-iommu.c

490

ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val,

drivers/iommu/rockchip-iommu.c

494

for (i = 0; i < iommu->num_mmu; i++)

drivers/iommu/rockchip-iommu.c

495

dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n",

drivers/iommu/rockchip-iommu.c

496

rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));

drivers/iommu/rockchip-iommu.c

501

static int rk_iommu_force_reset(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

507

if (iommu->reset_disabled)

drivers/iommu/rockchip-iommu.c

514

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

516

rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr);

drivers/iommu/rockchip-iommu.c

518

if (dte_addr != rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR)) {

drivers/iommu/rockchip-iommu.c

519

dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n");

drivers/iommu/rockchip-iommu.c

524

rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET);

drivers/iommu/rockchip-iommu.c

526

ret = readx_poll_timeout(rk_iommu_is_reset_done, iommu, val,

drivers/iommu/rockchip-iommu.c

530

dev_err(iommu->dev, "FORCE_RESET command timed out\n");

drivers/iommu/rockchip-iommu.c

537

static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)

drivers/iommu/rockchip-iommu.c

539

void __iomem *base = iommu->bases[index];

drivers/iommu/rockchip-iommu.c

576

dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n",

drivers/iommu/rockchip-iommu.c

578

dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n",

drivers/iommu/rockchip-iommu.c

586

struct rk_iommu *iommu = dev_id;

drivers/iommu/rockchip-iommu.c

593

err = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/rockchip-iommu.c

597

if (WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)))

drivers/iommu/rockchip-iommu.c

600

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

601

int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS);

drivers/iommu/rockchip-iommu.c

606

iova = rk_iommu_read(iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR);

drivers/iommu/rockchip-iommu.c

611

status = rk_iommu_read(iommu->bases[i], RK_MMU_STATUS);

drivers/iommu/rockchip-iommu.c

615

dev_err(iommu->dev, "Page fault at %pad of type %s\n",

drivers/iommu/rockchip-iommu.c

619

log_iova(iommu, i, iova);

drivers/iommu/rockchip-iommu.c

626

if (iommu->domain != &rk_identity_domain)

drivers/iommu/rockchip-iommu.c

627

report_iommu_fault(iommu->domain, iommu->dev, iova,

drivers/iommu/rockchip-iommu.c

630

dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n");

drivers/iommu/rockchip-iommu.c

632

rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);

drivers/iommu/rockchip-iommu.c

633

rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE);

drivers/iommu/rockchip-iommu.c

637

dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova);

drivers/iommu/rockchip-iommu.c

640

dev_err(iommu->dev, "unexpected int_status: %#08x\n",

drivers/iommu/rockchip-iommu.c

643

rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status);

drivers/iommu/rockchip-iommu.c

646

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

649

pm_runtime_put(iommu->dev);

drivers/iommu/rockchip-iommu.c

690

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

693

iommu = list_entry(pos, struct rk_iommu, node);

drivers/iommu/rockchip-iommu.c

696

ret = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/rockchip-iommu.c

700

WARN_ON(clk_bulk_enable(iommu->num_clocks,

drivers/iommu/rockchip-iommu.c

701

iommu->clocks));

drivers/iommu/rockchip-iommu.c

702

rk_iommu_zap_lines(iommu, iova, size);

drivers/iommu/rockchip-iommu.c

703

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

704

pm_runtime_put(iommu->dev);

drivers/iommu/rockchip-iommu.c

909

return data ? data->iommu : NULL;

drivers/iommu/rockchip-iommu.c

913

static void rk_iommu_disable(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

918

WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks));

drivers/iommu/rockchip-iommu.c

919

rk_iommu_enable_stall(iommu);

drivers/iommu/rockchip-iommu.c

920

rk_iommu_disable_paging(iommu);

drivers/iommu/rockchip-iommu.c

921

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

922

rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0);

drivers/iommu/rockchip-iommu.c

923

rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0);

drivers/iommu/rockchip-iommu.c

925

rk_iommu_disable_stall(iommu);

drivers/iommu/rockchip-iommu.c

926

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

930

static int rk_iommu_enable(struct rk_iommu *iommu)

drivers/iommu/rockchip-iommu.c

932

struct iommu_domain *domain = iommu->domain;

drivers/iommu/rockchip-iommu.c

937

ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

941

ret = rk_iommu_enable_stall(iommu);

drivers/iommu/rockchip-iommu.c

945

ret = rk_iommu_force_reset(iommu);

drivers/iommu/rockchip-iommu.c

949

for (i = 0; i < iommu->num_mmu; i++) {

drivers/iommu/rockchip-iommu.c

950

rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,

drivers/iommu/rockchip-iommu.c

952

rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);

drivers/iommu/rockchip-iommu.c

953

rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);

drivers/iommu/rockchip-iommu.c

956

auto_gate = rk_iommu_read(iommu->bases[i], RK_MMU_AUTO_GATING);

drivers/iommu/rockchip-iommu.c

958

rk_iommu_write(iommu->bases[i], RK_MMU_AUTO_GATING, auto_gate);

drivers/iommu/rockchip-iommu.c

961

ret = rk_iommu_enable_paging(iommu);

drivers/iommu/rockchip-iommu.c

964

rk_iommu_disable_stall(iommu);

drivers/iommu/rockchip-iommu.c

966

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/rockchip-iommu.c

974

struct rk_iommu *iommu;

drivers/iommu/rockchip-iommu.c

980

iommu = rk_iommu_from_dev(dev);

drivers/iommu/rockchip-iommu.c

981

if (!iommu)

drivers/iommu/rockchip-iommu.c

984

rk_domain = to_rk_domain(iommu->domain);

drivers/iommu/rockchip-iommu.c

988

if (iommu->domain == identity_domain)

drivers/iommu/rockchip-iommu.c

991

iommu->domain = identity_domain;

drivers/iommu/rockchip-iommu.c

994

list_del_init(&iommu->node);

drivers/iommu/rockchip-iommu.c

997

ret = pm_runtime_get_if_in_use(iommu->dev);

drivers/iommu/s390-iommu.c

780

dev->iommu->shadow_on_flush = 1;

drivers/iommu/sprd-iommu.c

393

return &sdev->iommu;

drivers/iommu/sprd-iommu.c

486

ret = iommu_device_sysfs_add(&sdev->iommu, dev, NULL, dev_name(dev));

drivers/iommu/sprd-iommu.c

490

ret = iommu_device_register(&sdev->iommu, &sprd_iommu_ops, dev);

drivers/iommu/sprd-iommu.c

510

iommu_device_unregister(&sdev->iommu);

drivers/iommu/sprd-iommu.c

512

iommu_device_sysfs_remove(&sdev->iommu);

drivers/iommu/sprd-iommu.c

525

iommu_device_sysfs_remove(&sdev->iommu);

drivers/iommu/sprd-iommu.c

526

iommu_device_unregister(&sdev->iommu);

drivers/iommu/sprd-iommu.c

72

struct iommu_device iommu;

drivers/iommu/sun50i-iommu.c

1001

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

1004

iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/sun50i-iommu.c

1005

if (!iommu)

drivers/iommu/sun50i-iommu.c

1007

spin_lock_init(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

1008

iommu->domain = &sun50i_iommu_identity_domain;

drivers/iommu/sun50i-iommu.c

1009

platform_set_drvdata(pdev, iommu);

drivers/iommu/sun50i-iommu.c

101

struct iommu_device iommu;

drivers/iommu/sun50i-iommu.c

1010

iommu->dev = &pdev->dev;

drivers/iommu/sun50i-iommu.c

1012

iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),

drivers/iommu/sun50i-iommu.c

1016

if (!iommu->pt_pool)

drivers/iommu/sun50i-iommu.c

1019

iommu->base = devm_platform_ioremap_resource(pdev, 0);

drivers/iommu/sun50i-iommu.c

1020

if (IS_ERR(iommu->base)) {

drivers/iommu/sun50i-iommu.c

1021

ret = PTR_ERR(iommu->base);

drivers/iommu/sun50i-iommu.c

1031

iommu->clk = devm_clk_get(&pdev->dev, NULL);

drivers/iommu/sun50i-iommu.c

1032

if (IS_ERR(iommu->clk)) {

drivers/iommu/sun50i-iommu.c

1034

ret = PTR_ERR(iommu->clk);

drivers/iommu/sun50i-iommu.c

1038

iommu->reset = devm_reset_control_get(&pdev->dev, NULL);

drivers/iommu/sun50i-iommu.c

1039

if (IS_ERR(iommu->reset)) {

drivers/iommu/sun50i-iommu.c

1041

ret = PTR_ERR(iommu->reset);

drivers/iommu/sun50i-iommu.c

1045

ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,

drivers/iommu/sun50i-iommu.c

1050

ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);

drivers/iommu/sun50i-iommu.c

1055

dev_name(&pdev->dev), iommu);

drivers/iommu/sun50i-iommu.c

1062

iommu_device_unregister(&iommu->iommu);

drivers/iommu/sun50i-iommu.c

1065

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/sun50i-iommu.c

1068

kmem_cache_destroy(iommu->pt_pool);

drivers/iommu/sun50i-iommu.c

125

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

138

static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)

drivers/iommu/sun50i-iommu.c

140

return readl(iommu->base + offset);

drivers/iommu/sun50i-iommu.c

143

static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)

drivers/iommu/sun50i-iommu.c

145

writel(value, iommu->base + offset);

drivers/iommu/sun50i-iommu.c

294

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

298

dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);

drivers/iommu/sun50i-iommu.c

301

static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

307

iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);

drivers/iommu/sun50i-iommu.c

308

iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));

drivers/iommu/sun50i-iommu.c

309

iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

312

ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

315

dev_warn(iommu->dev, "TLB invalidation timed out!\n");

drivers/iommu/sun50i-iommu.c

318

static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

324

iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);

drivers/iommu/sun50i-iommu.c

325

iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

328

ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,

drivers/iommu/sun50i-iommu.c

331

dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");

drivers/iommu/sun50i-iommu.c

334

static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

337

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

339

iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);

drivers/iommu/sun50i-iommu.c

341

sun50i_iommu_zap_iova(iommu, iova);

drivers/iommu/sun50i-iommu.c

342

sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);

drivers/iommu/sun50i-iommu.c

344

sun50i_iommu_zap_iova(iommu, iova + size);

drivers/iommu/sun50i-iommu.c

345

sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);

drivers/iommu/sun50i-iommu.c

347

sun50i_iommu_zap_ptw_cache(iommu, iova);

drivers/iommu/sun50i-iommu.c

348

sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);

drivers/iommu/sun50i-iommu.c

350

sun50i_iommu_zap_ptw_cache(iommu, iova + size);

drivers/iommu/sun50i-iommu.c

351

sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);

drivers/iommu/sun50i-iommu.c

354

iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);

drivers/iommu/sun50i-iommu.c

357

static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

362

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

364

iommu_write(iommu,

drivers/iommu/sun50i-iommu.c

375

ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,

drivers/iommu/sun50i-iommu.c

379

dev_warn(iommu->dev, "TLB Flush timed out!\n");

drivers/iommu/sun50i-iommu.c

387

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

398

if (!iommu)

drivers/iommu/sun50i-iommu.c

401

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

402

sun50i_iommu_flush_all_tlb(iommu);

drivers/iommu/sun50i-iommu.c

403

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

410

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

413

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

414

sun50i_iommu_zap_range(iommu, iova, size);

drivers/iommu/sun50i-iommu.c

415

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

426

static int sun50i_iommu_enable(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

432

if (!iommu->domain)

drivers/iommu/sun50i-iommu.c

435

sun50i_domain = to_sun50i_domain(iommu->domain);

drivers/iommu/sun50i-iommu.c

437

ret = reset_control_deassert(iommu->reset);

drivers/iommu/sun50i-iommu.c

441

ret = clk_prepare_enable(iommu->clk);

drivers/iommu/sun50i-iommu.c

445

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

447

iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);

drivers/iommu/sun50i-iommu.c

448

iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,

drivers/iommu/sun50i-iommu.c

455

iommu_write(iommu, IOMMU_BYPASS_REG, 0);

drivers/iommu/sun50i-iommu.c

456

iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);

drivers/iommu/sun50i-iommu.c

457

iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),

drivers/iommu/sun50i-iommu.c

471

iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),

drivers/iommu/sun50i-iommu.c

479

iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),

drivers/iommu/sun50i-iommu.c

487

ret = sun50i_iommu_flush_all_tlb(iommu);

drivers/iommu/sun50i-iommu.c

489

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

493

iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);

drivers/iommu/sun50i-iommu.c

494

iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);

drivers/iommu/sun50i-iommu.c

496

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

501

clk_disable_unprepare(iommu->clk);

drivers/iommu/sun50i-iommu.c

504

reset_control_assert(iommu->reset);

drivers/iommu/sun50i-iommu.c

509

static void sun50i_iommu_disable(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

513

spin_lock_irqsave(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

515

iommu_write(iommu, IOMMU_ENABLE_REG, 0);

drivers/iommu/sun50i-iommu.c

516

iommu_write(iommu, IOMMU_TTB_REG, 0);

drivers/iommu/sun50i-iommu.c

518

spin_unlock_irqrestore(&iommu->iommu_lock, flags);

drivers/iommu/sun50i-iommu.c

520

clk_disable_unprepare(iommu->clk);

drivers/iommu/sun50i-iommu.c

521

reset_control_assert(iommu->reset);

drivers/iommu/sun50i-iommu.c

524

static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

530

page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);

drivers/iommu/sun50i-iommu.c

534

pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);

drivers/iommu/sun50i-iommu.c

535

if (dma_mapping_error(iommu->dev, pt_dma)) {

drivers/iommu/sun50i-iommu.c

536

dev_err(iommu->dev, "Couldn't map L2 Page Table\n");

drivers/iommu/sun50i-iommu.c

537

kmem_cache_free(iommu->pt_pool, page_table);

drivers/iommu/sun50i-iommu.c

547

static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

552

dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);

drivers/iommu/sun50i-iommu.c

553

kmem_cache_free(iommu->pt_pool, page_table);

drivers/iommu/sun50i-iommu.c

559

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

572

page_table = sun50i_iommu_alloc_page_table(iommu, gfp);

drivers/iommu/sun50i-iommu.c

586

sun50i_iommu_free_page_table(iommu, drop_pt);

drivers/iommu/sun50i-iommu.c

600

struct sun50i_iommu *iommu = sun50i_domain->iommu;

drivers/iommu/sun50i-iommu.c

608

dev_warn_once(iommu->dev,

drivers/iommu/sun50i-iommu.c

623

dev_err(iommu->dev,

drivers/iommu/sun50i-iommu.c

724

static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

727

iommu->domain = &sun50i_domain->domain;

drivers/iommu/sun50i-iommu.c

728

sun50i_domain->iommu = iommu;

drivers/iommu/sun50i-iommu.c

730

sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,

drivers/iommu/sun50i-iommu.c

732

if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {

drivers/iommu/sun50i-iommu.c

733

dev_err(iommu->dev, "Couldn't map L1 Page Table\n");

drivers/iommu/sun50i-iommu.c

737

return sun50i_iommu_enable(iommu);

drivers/iommu/sun50i-iommu.c

740

static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

761

sun50i_iommu_free_page_table(iommu, page_table);

drivers/iommu/sun50i-iommu.c

765

sun50i_iommu_disable(iommu);

drivers/iommu/sun50i-iommu.c

767

dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),

drivers/iommu/sun50i-iommu.c

770

iommu->domain = NULL;

drivers/iommu/sun50i-iommu.c

777

struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);

drivers/iommu/sun50i-iommu.c

782

if (iommu->domain == identity_domain)

drivers/iommu/sun50i-iommu.c

785

sun50i_domain = to_sun50i_domain(iommu->domain);

drivers/iommu/sun50i-iommu.c

787

sun50i_iommu_detach_domain(iommu, sun50i_domain);

drivers/iommu/sun50i-iommu.c

805

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

807

iommu = sun50i_iommu_from_dev(dev);

drivers/iommu/sun50i-iommu.c

808

if (!iommu)

drivers/iommu/sun50i-iommu.c

815

if (iommu->domain == domain)

drivers/iommu/sun50i-iommu.c

820

sun50i_iommu_attach_domain(iommu, sun50i_domain);

drivers/iommu/sun50i-iommu.c

827

struct sun50i_iommu *iommu;

drivers/iommu/sun50i-iommu.c

829

iommu = sun50i_iommu_from_dev(dev);

drivers/iommu/sun50i-iommu.c

830

if (!iommu)

drivers/iommu/sun50i-iommu.c

833

return &iommu->iommu;

drivers/iommu/sun50i-iommu.c

867

static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

871

dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",

drivers/iommu/sun50i-iommu.c

874

if (iommu->domain)

drivers/iommu/sun50i-iommu.c

875

report_iommu_fault(iommu->domain, iommu->dev, iova, prot);

drivers/iommu/sun50i-iommu.c

877

dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");

drivers/iommu/sun50i-iommu.c

879

sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);

drivers/iommu/sun50i-iommu.c

882

static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,

drivers/iommu/sun50i-iommu.c

890

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

892

iova = iommu_read(iommu, addr_reg);

drivers/iommu/sun50i-iommu.c

893

blame = iommu_read(iommu, blame_reg);

drivers/iommu/sun50i-iommu.c

901

sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);

drivers/iommu/sun50i-iommu.c

906

static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)

drivers/iommu/sun50i-iommu.c

914

assert_spin_locked(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

916

blame = iommu_read(iommu, IOMMU_INT_STA_REG);

drivers/iommu/sun50i-iommu.c

918

iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));

drivers/iommu/sun50i-iommu.c

919

aci = sun50i_get_pte_aci(iommu_read(iommu,

drivers/iommu/sun50i-iommu.c

956

sun50i_iommu_report_fault(iommu, master, iova, dir);

drivers/iommu/sun50i-iommu.c

964

struct sun50i_iommu *iommu = dev_id;

drivers/iommu/sun50i-iommu.c

966

spin_lock(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

968

status = iommu_read(iommu, IOMMU_INT_STA_REG);

drivers/iommu/sun50i-iommu.c

970

spin_unlock(&iommu->iommu_lock);

drivers/iommu/sun50i-iommu.c

974

l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);

drivers/iommu/sun50i-iommu.c

975

l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);

drivers/iommu/sun50i-iommu.c

978

sun50i_iommu_handle_pt_irq(iommu,

drivers/iommu/sun50i-iommu.c

982

sun50i_iommu_handle_pt_irq(iommu,

drivers/iommu/sun50i-iommu.c

986

sun50i_iommu_handle_perm_irq(iommu);

drivers/iommu/sun50i-iommu.c

988

iommu_write(iommu, IOMMU_INT_CLR_REG, status);

drivers/iommu/sun50i-iommu.c

991

iommu_write(iommu, IOMMU_RESET_REG, ~resets);

drivers/iommu/sun50i-iommu.c

992

iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);

drivers/iommu/sun50i-iommu.c

994

spin_unlock(&iommu->iommu_lock);

drivers/iommu/tegra-smmu.c

1169

err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));

drivers/iommu/tegra-smmu.c

1173

err = iommu_device_register(&smmu->iommu, &tegra_smmu_ops, dev);

drivers/iommu/tegra-smmu.c

1175

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/tegra-smmu.c

1187

iommu_device_unregister(&smmu->iommu);

drivers/iommu/tegra-smmu.c

1188

iommu_device_sysfs_remove(&smmu->iommu);

drivers/iommu/tegra-smmu.c

51

struct iommu_device iommu; /* IOMMU Core code handle */

drivers/iommu/tegra-smmu.c

843

const struct iommu_ops *ops = smmu->iommu.ops;

drivers/iommu/tegra-smmu.c

889

return &smmu->iommu;

drivers/iommu/virtio-iommu.c

1047

return &viommu->iommu;

drivers/iommu/virtio-iommu.c

1232

ret = iommu_device_sysfs_add(&viommu->iommu, dev, NULL, "%s",

drivers/iommu/virtio-iommu.c

1239

iommu_device_register(&viommu->iommu, &viommu_ops, parent_dev);

drivers/iommu/virtio-iommu.c

1257

iommu_device_sysfs_remove(&viommu->iommu);

drivers/iommu/virtio-iommu.c

1258

iommu_device_unregister(&viommu->iommu);

drivers/iommu/virtio-iommu.c

35

struct iommu_device iommu;

drivers/iommu/vsi-iommu.c

194

struct vsi_iommu *iommu = dev_id;

drivers/iommu/vsi-iommu.c

199

if (pm_runtime_resume_and_get(iommu->dev) < 0)

drivers/iommu/vsi-iommu.c

202

spin_lock_irqsave(&iommu->lock, flags);

drivers/iommu/vsi-iommu.c

204

status = readl(iommu->regs + VSI_MMU_STATUS_BASE);

drivers/iommu/vsi-iommu.c

206

dev_err(iommu->dev, "unexpected int_status=%08x\n", status);

drivers/iommu/vsi-iommu.c

207

iova = readl(iommu->regs + VSI_MMU_PAGE_FAULT_ADDR);

drivers/iommu/vsi-iommu.c

208

report_iommu_fault(iommu->domain, iommu->dev, iova, status);

drivers/iommu/vsi-iommu.c

210

writel(0, iommu->regs + VSI_MMU_STATUS_BASE);

drivers/iommu/vsi-iommu.c

212

spin_unlock_irqrestore(&iommu->lock, flags);

drivers/iommu/vsi-iommu.c

213

pm_runtime_put_autosuspend(iommu->dev);

drivers/iommu/vsi-iommu.c

231

struct vsi_iommu *iommu = dev_iommu_priv_get(dev);

drivers/iommu/vsi-iommu.c

238

vsi_domain->dev = iommu->dev;

drivers/iommu/vsi-iommu.c

369

struct vsi_iommu *iommu;

drivers/iommu/vsi-iommu.c

371

list_for_each_entry(iommu, &vsi_domain->iommus, node) {

drivers/iommu/vsi-iommu.c

372

if (pm_runtime_get(iommu->dev) < 0)

drivers/iommu/vsi-iommu.c

375

spin_lock(&iommu->lock);

drivers/iommu/vsi-iommu.c

377

if (iommu->enable) {

drivers/iommu/vsi-iommu.c

378

writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);

drivers/iommu/vsi-iommu.c

379

writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);

drivers/iommu/vsi-iommu.c

382

spin_unlock(&iommu->lock);

drivers/iommu/vsi-iommu.c

384

pm_runtime_put_autosuspend(iommu->dev);

drivers/iommu/vsi-iommu.c

46

struct iommu_device iommu;

drivers/iommu/vsi-iommu.c

493

static void vsi_iommu_disable(struct vsi_iommu *iommu)

drivers/iommu/vsi-iommu.c

495

writel(0, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);

drivers/iommu/vsi-iommu.c

496

iommu->enable = false;

drivers/iommu/vsi-iommu.c

502

struct vsi_iommu *iommu = dev_iommu_priv_get(dev);

drivers/iommu/vsi-iommu.c

507

ret = pm_runtime_resume_and_get(iommu->dev);

drivers/iommu/vsi-iommu.c

512

spin_lock(&iommu->lock);

drivers/iommu/vsi-iommu.c

513

if (iommu->domain == domain)

drivers/iommu/vsi-iommu.c

516

vsi_iommu_disable(iommu);

drivers/iommu/vsi-iommu.c

517

list_del_init(&iommu->node);

drivers/iommu/vsi-iommu.c

519

iommu->domain = domain;

drivers/iommu/vsi-iommu.c

522

spin_unlock(&iommu->lock);

drivers/iommu/vsi-iommu.c

524

pm_runtime_put_autosuspend(iommu->dev);

drivers/iommu/vsi-iommu.c

537

static void vsi_iommu_enable(struct vsi_iommu *iommu, struct iommu_domain *domain)

drivers/iommu/vsi-iommu.c

544

writel(vsi_domain->pta_dma, iommu->regs + VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE);

drivers/iommu/vsi-iommu.c

545

writel(VSI_MMU_OUT_OF_BOUND, iommu->regs + VSI_MMU_CONFIG1_BASE);

drivers/iommu/vsi-iommu.c

546

writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_EXCEPTION_BASE);

drivers/iommu/vsi-iommu.c

547

writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);

drivers/iommu/vsi-iommu.c

548

iommu->enable = true;

drivers/iommu/vsi-iommu.c

554

struct vsi_iommu *iommu = dev_iommu_priv_get(dev);

drivers/iommu/vsi-iommu.c

559

ret = pm_runtime_resume_and_get(iommu->dev);

drivers/iommu/vsi-iommu.c

564

spin_lock(&iommu->lock);

drivers/iommu/vsi-iommu.c

566

vsi_iommu_enable(iommu, domain);

drivers/iommu/vsi-iommu.c

567

writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);

drivers/iommu/vsi-iommu.c

568

writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);

drivers/iommu/vsi-iommu.c

570

list_del_init(&iommu->node);

drivers/iommu/vsi-iommu.c

571

list_add_tail(&iommu->node, &vsi_domain->iommus);

drivers/iommu/vsi-iommu.c

573

iommu->domain = domain;

drivers/iommu/vsi-iommu.c

575

spin_unlock(&iommu->lock);

drivers/iommu/vsi-iommu.c

577

pm_runtime_put_autosuspend(iommu->dev);

drivers/iommu/vsi-iommu.c

619

struct vsi_iommu *iommu = vsi_iommu_get_from_dev(dev);

drivers/iommu/vsi-iommu.c

622

link = device_link_add(dev, iommu->dev,

drivers/iommu/vsi-iommu.c

625

dev_err(dev, "Unable to link %s\n", dev_name(iommu->dev));

drivers/iommu/vsi-iommu.c

627

dev_iommu_priv_set(dev, iommu);

drivers/iommu/vsi-iommu.c

628

return &iommu->iommu;

drivers/iommu/vsi-iommu.c

633

struct vsi_iommu *iommu = dev_iommu_priv_get(dev);

drivers/iommu/vsi-iommu.c

635

device_link_remove(dev, iommu->dev);

drivers/iommu/vsi-iommu.c

672

struct vsi_iommu *iommu;

drivers/iommu/vsi-iommu.c

675

iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);

drivers/iommu/vsi-iommu.c

676

if (!iommu)

drivers/iommu/vsi-iommu.c

679

iommu->dev = dev;

drivers/iommu/vsi-iommu.c

680

spin_lock_init(&iommu->lock);

drivers/iommu/vsi-iommu.c

681

INIT_LIST_HEAD(&iommu->node);

drivers/iommu/vsi-iommu.c

683

iommu->regs = devm_platform_ioremap_resource(pdev, 0);

drivers/iommu/vsi-iommu.c

684

if (IS_ERR(iommu->regs))

drivers/iommu/vsi-iommu.c

687

iommu->num_clocks = devm_clk_bulk_get_all(dev, &iommu->clocks);

drivers/iommu/vsi-iommu.c

688

if (iommu->num_clocks < 0)

drivers/iommu/vsi-iommu.c

689

return iommu->num_clocks;

drivers/iommu/vsi-iommu.c

691

err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);

drivers/iommu/vsi-iommu.c

695

iommu->irq = platform_get_irq(pdev, 0);

drivers/iommu/vsi-iommu.c

696

if (iommu->irq < 0)

drivers/iommu/vsi-iommu.c

697

return iommu->irq;

drivers/iommu/vsi-iommu.c

699

err = devm_request_irq(iommu->dev, iommu->irq, vsi_iommu_irq,

drivers/iommu/vsi-iommu.c

700

IRQF_SHARED, dev_name(dev), iommu);

drivers/iommu/vsi-iommu.c

705

platform_set_drvdata(pdev, iommu);

drivers/iommu/vsi-iommu.c

711

err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, "%s",

drivers/iommu/vsi-iommu.c

716

err = iommu_device_register(&iommu->iommu, &vsi_iommu_ops, dev);

drivers/iommu/vsi-iommu.c

723

iommu_device_sysfs_remove(&iommu->iommu);

drivers/iommu/vsi-iommu.c

727

clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);

drivers/iommu/vsi-iommu.c

733

struct vsi_iommu *iommu = platform_get_drvdata(pdev);

drivers/iommu/vsi-iommu.c

735

disable_irq(iommu->irq);

drivers/iommu/vsi-iommu.c

741

struct vsi_iommu *iommu = dev_get_drvdata(dev);

drivers/iommu/vsi-iommu.c

743

vsi_iommu_disable(iommu);

drivers/iommu/vsi-iommu.c

745

clk_bulk_disable(iommu->num_clocks, iommu->clocks);

drivers/iommu/vsi-iommu.c

752

struct vsi_iommu *iommu = dev_get_drvdata(dev);

drivers/iommu/vsi-iommu.c

756

ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);

drivers/iommu/vsi-iommu.c

760

if (iommu->domain) {

drivers/iommu/vsi-iommu.c

761

struct vsi_iommu_domain *vsi_domain = to_vsi_domain(iommu->domain);

drivers/iommu/vsi-iommu.c

764

spin_lock(&iommu->lock);

drivers/iommu/vsi-iommu.c

765

vsi_iommu_enable(iommu, iommu->domain);

drivers/iommu/vsi-iommu.c

766

spin_unlock(&iommu->lock);

drivers/media/platform/qcom/venus/firmware.c

142

struct iommu_domain *iommu;

drivers/media/platform/qcom/venus/firmware.c

150

iommu = core->fw.iommu_domain;

drivers/media/platform/qcom/venus/firmware.c

153

ret = iommu_map(iommu, VENUS_FW_START_ADDR, mem_phys, mem_size,

drivers/media/platform/qcom/venus/firmware.c

168

struct iommu_domain *iommu;

drivers/media/platform/qcom/venus/firmware.c

187

iommu = core->fw.iommu_domain;

drivers/media/platform/qcom/venus/firmware.c

189

if (core->fw.mapped_mem_size && iommu) {

drivers/media/platform/qcom/venus/firmware.c

190

unmapped = iommu_unmap(iommu, VENUS_FW_START_ADDR, mapped);

drivers/media/platform/qcom/venus/firmware.c

369

struct iommu_domain *iommu;

drivers/media/platform/qcom/venus/firmware.c

374

iommu = core->fw.iommu_domain;

drivers/media/platform/qcom/venus/firmware.c

376

iommu_detach_device(iommu, core->fw.dev);

drivers/media/platform/qcom/venus/firmware.c

379

iommu_domain_free(iommu);

drivers/net/wireless/ath/ath10k/snoc.c

1697

struct iommu_domain *iommu;

drivers/net/wireless/ath/ath10k/snoc.c

1703

iommu = ar_snoc->fw.iommu_domain;

drivers/net/wireless/ath/ath10k/snoc.c

1705

unmapped_size = iommu_unmap(iommu, ar_snoc->fw.fw_start_addr,

drivers/net/wireless/ath/ath10k/snoc.c

1711

iommu_detach_device(iommu, ar_snoc->fw.dev);

drivers/net/wireless/ath/ath10k/snoc.c

1712

iommu_domain_free(iommu);

drivers/net/wireless/ath/ath11k/ahb.c

1076

struct iommu_domain *iommu;

drivers/net/wireless/ath/ath11k/ahb.c

1088

iommu = ab_ahb->fw.iommu_domain;

drivers/net/wireless/ath/ath11k/ahb.c

1090

unmapped_size = iommu_unmap(iommu, ab_ahb->fw.msa_paddr, ab_ahb->fw.msa_size);

drivers/net/wireless/ath/ath11k/ahb.c

1095

unmapped_size = iommu_unmap(iommu, ab_ahb->fw.ce_paddr, ab_ahb->fw.ce_size);

drivers/net/wireless/ath/ath11k/ahb.c

1100

iommu_detach_device(iommu, ab_ahb->fw.dev);

drivers/net/wireless/ath/ath11k/ahb.c

1101

iommu_domain_free(iommu);

drivers/parisc/ccio-dma.c

1557

hba->iommu = ioc;

drivers/parisc/dino.c

1003

dino_dev->hba.iommu = ccio_get_iommu(dev);

drivers/parisc/eisa.c

304

eisa_dev.hba.iommu = ccio_get_iommu(dev);

drivers/parisc/iommu.h

35

return pdata->iommu;

drivers/parisc/lba_pci.c

1562

lba_dev->hba.iommu = sba_get_iommu(dev); /* get iommu data */

drivers/vdpa/vdpa_sim/vdpa_sim.c

150

vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],

drivers/vdpa/vdpa_sim/vdpa_sim.c

156

vhost_iotlb_reset(&vdpasim->iommu[i]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

157

vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,

drivers/vdpa/vdpa_sim/vdpa_sim.c

253

vdpasim->iommu = kmalloc_objs(*vdpasim->iommu, vdpasim->dev_attr.nas);

drivers/vdpa/vdpa_sim/vdpa_sim.c

254

if (!vdpasim->iommu)

drivers/vdpa/vdpa_sim/vdpa_sim.c

263

vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);

drivers/vdpa/vdpa_sim/vdpa_sim.c

264

vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 0,

drivers/vdpa/vdpa_sim/vdpa_sim.c

270

vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],

drivers/vdpa/vdpa_sim/vdpa_sim.c

604

struct vhost_iotlb *iommu;

drivers/vdpa/vdpa_sim/vdpa_sim.c

607

iommu = &vdpasim->iommu[asid];

drivers/vdpa/vdpa_sim/vdpa_sim.c

613

vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,

drivers/vdpa/vdpa_sim/vdpa_sim.c

626

struct vhost_iotlb *iommu;

drivers/vdpa/vdpa_sim/vdpa_sim.c

635

iommu = &vdpasim->iommu[asid];

drivers/vdpa/vdpa_sim/vdpa_sim.c

636

vhost_iotlb_reset(iommu);

drivers/vdpa/vdpa_sim/vdpa_sim.c

641

ret = vhost_iotlb_add_range(iommu, map->start,

drivers/vdpa/vdpa_sim/vdpa_sim.c

650

vhost_iotlb_reset(iommu);

drivers/vdpa/vdpa_sim/vdpa_sim.c

665

vhost_iotlb_reset(&vdpasim->iommu[asid]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

666

vhost_iotlb_add_range(&vdpasim->iommu[asid], 0, ULONG_MAX,

drivers/vdpa/vdpa_sim/vdpa_sim.c

710

vhost_iotlb_reset(&vdpasim->iommu[asid]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

713

ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova,

drivers/vdpa/vdpa_sim/vdpa_sim.c

729

vhost_iotlb_reset(&vdpasim->iommu[asid]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

734

vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1);

drivers/vdpa/vdpa_sim/vdpa_sim.c

756

vhost_iotlb_reset(&vdpasim->iommu[i]);

drivers/vdpa/vdpa_sim/vdpa_sim.c

757

kfree(vdpasim->iommu);

drivers/vdpa/vdpa_sim/vdpa_sim.h

68

struct vhost_iotlb *iommu;

drivers/vfio/vfio_iommu_type1.c

1010

dma = vfio_find_dma(iommu, iova, PAGE_SIZE);

drivers/vfio/vfio_iommu_type1.c

1015

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1022

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

1029

if (WARN_ON(!iommu->v2))

drivers/vfio/vfio_iommu_type1.c

1039

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1041

do_accounting = list_empty(&iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

1046

dma = vfio_find_dma(iommu, iova, PAGE_SIZE);

drivers/vfio/vfio_iommu_type1.c

1053

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1147

static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,

drivers/vfio/vfio_iommu_type1.c

1160

if (list_empty(&iommu->domain_list))

drivers/vfio/vfio_iommu_type1.c

1170

domain = d = list_first_entry(&iommu->domain_list,

drivers/vfio/vfio_iommu_type1.c

1173

list_for_each_entry_continue(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

1233

static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)

drivers/vfio/vfio_iommu_type1.c

1236

vfio_unmap_unpin(iommu, dma, true);

drivers/vfio/vfio_iommu_type1.c

1237

vfio_unlink_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1242

iommu->vaddr_invalid_count--;

drivers/vfio/vfio_iommu_type1.c

1244

iommu->dma_avail++;

drivers/vfio/vfio_iommu_type1.c

1247

static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

1251

iommu->pgsize_bitmap = ULONG_MAX;

drivers/vfio/vfio_iommu_type1.c

1253

list_for_each_entry(domain, &iommu->domain_list, next)

drivers/vfio/vfio_iommu_type1.c

1254

iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap;

drivers/vfio/vfio_iommu_type1.c

1264

if (iommu->pgsize_bitmap & ~PAGE_MASK) {

drivers/vfio/vfio_iommu_type1.c

1265

iommu->pgsize_bitmap &= PAGE_MASK;

drivers/vfio/vfio_iommu_type1.c

1266

iommu->pgsize_bitmap |= PAGE_SIZE;

drivers/vfio/vfio_iommu_type1.c

1270

static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1285

if (iommu->num_non_pinned_groups && dma->iommu_mapped)

drivers/vfio/vfio_iommu_type1.c

1307

static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1321

dma = vfio_find_dma(iommu, iova, 1);

drivers/vfio/vfio_iommu_type1.c

1325

dma = vfio_find_dma(iommu, iova_end, 1);

drivers/vfio/vfio_iommu_type1.c

1329

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

1338

ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);

drivers/vfio/vfio_iommu_type1.c

1367

static void vfio_notify_dma_unmap(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1372

if (list_empty(&iommu->device_list))

drivers/vfio/vfio_iommu_type1.c

1381

mutex_lock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

1382

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1384

list_for_each_entry(device, &iommu->device_list, iommu_entry)

drivers/vfio/vfio_iommu_type1.c

1387

mutex_unlock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

1388

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1391

static int vfio_dma_do_unmap(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1406

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1409

if (invalidate_vaddr && !list_empty(&iommu->emulated_iommu_groups)) {

drivers/vfio/vfio_iommu_type1.c

1419

pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

1441

(!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) {

drivers/vfio/vfio_iommu_type1.c

1478

if (iommu->v2 && !unmap_all) {

drivers/vfio/vfio_iommu_type1.c

1479

dma = vfio_find_dma(iommu, iova, 1);

drivers/vfio/vfio_iommu_type1.c

1483

dma = vfio_find_dma(iommu, iova_end, 1);

drivers/vfio/vfio_iommu_type1.c

1489

n = first_n = vfio_find_dma_first_node(iommu, iova, iova_end);

drivers/vfio/vfio_iommu_type1.c

1496

if (!iommu->v2 && iova > dma->iova)

drivers/vfio/vfio_iommu_type1.c

1507

iommu->vaddr_invalid_count--;

drivers/vfio/vfio_iommu_type1.c

1514

iommu->vaddr_invalid_count++;

drivers/vfio/vfio_iommu_type1.c

1528

vfio_notify_dma_unmap(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1533

ret = update_user_bitmap(bitmap->data, iommu, dma,

drivers/vfio/vfio_iommu_type1.c

1541

vfio_remove_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1545

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1553

static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova,

drivers/vfio/vfio_iommu_type1.c

1559

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

157

vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1572

list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

1580

static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,

drivers/vfio/vfio_iommu_type1.c

1605

ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,

drivers/vfio/vfio_iommu_type1.c

1622

vfio_remove_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1630

static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1633

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

165

static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

168

struct rb_node *node = iommu->dma_list.rb_node;

drivers/vfio/vfio_iommu_type1.c

1680

static int vfio_dma_do_map(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1713

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1715

pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

1724

dma = vfio_find_dma(iommu, iova, size);

drivers/vfio/vfio_iommu_type1.c

1737

iommu->vaddr_invalid_count--;

drivers/vfio/vfio_iommu_type1.c

1745

if (!iommu->dma_avail) {

drivers/vfio/vfio_iommu_type1.c

1750

if (!vfio_iommu_iova_dma_valid(iommu, iova, iova_end)) {

drivers/vfio/vfio_iommu_type1.c

1761

iommu->dma_avail--;

drivers/vfio/vfio_iommu_type1.c

1784

vfio_link_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1787

if (list_empty(&iommu->domain_list))

drivers/vfio/vfio_iommu_type1.c

1790

ret = vfio_pin_map_dma(iommu, dma, size);

drivers/vfio/vfio_iommu_type1.c

1792

if (!ret && iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

1795

vfio_remove_dma(iommu, dma);

drivers/vfio/vfio_iommu_type1.c

1799

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

1803

static int vfio_iommu_replay(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1813

if (!list_empty(&iommu->domain_list))

drivers/vfio/vfio_iommu_type1.c

1814

d = list_first_entry(&iommu->domain_list,

drivers/vfio/vfio_iommu_type1.c

1819

n = rb_first(&iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

186

static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1896

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

191

struct rb_node *node = iommu->dma_list.rb_node;

drivers/vfio/vfio_iommu_type1.c

1961

vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

1967

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

1973

list_for_each_entry(group, &iommu->emulated_iommu_groups, next)

drivers/vfio/vfio_iommu_type1.c

2035

static bool vfio_iommu_aper_conflict(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2039

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

2052

if (vfio_find_dma(iommu, first->start, start - first->start))

drivers/vfio/vfio_iommu_type1.c

2058

if (vfio_find_dma(iommu, end + 1, last->end - end))

drivers/vfio/vfio_iommu_type1.c

2109

static bool vfio_iommu_resv_conflict(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2119

if (vfio_find_dma(iommu, region->start, region->length))

drivers/vfio/vfio_iommu_type1.c

214

static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new)

drivers/vfio/vfio_iommu_type1.c

216

struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL;

drivers/vfio/vfio_iommu_type1.c

2198

static int vfio_iommu_iova_get_copy(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2201

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

2218

static void vfio_iommu_iova_insert_copy(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2221

struct list_head *iova = &iommu->iova_list;

drivers/vfio/vfio_iommu_type1.c

2239

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

2249

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2252

if (iommu->vaddr_invalid_count)

drivers/vfio/vfio_iommu_type1.c

2257

if (vfio_iommu_find_iommu_group(iommu, iommu_group))

drivers/vfio/vfio_iommu_type1.c

2267

list_add(&group->next, &iommu->emulated_iommu_groups);

drivers/vfio/vfio_iommu_type1.c

2302

if (vfio_iommu_aper_conflict(iommu, geo->aperture_start,

drivers/vfio/vfio_iommu_type1.c

2312

if (vfio_iommu_resv_conflict(iommu, &group_resv_regions)) {

drivers/vfio/vfio_iommu_type1.c

232

rb_insert_color(&new->node, &iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

2322

ret = vfio_iommu_iova_get_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

235

static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)

drivers/vfio/vfio_iommu_type1.c

2365

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

237

rb_erase(&old->node, &iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

2386

ret = vfio_iommu_replay(iommu, domain);

drivers/vfio/vfio_iommu_type1.c

2396

list_add(&domain->next, &iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

2397

vfio_update_pgsize_bitmap(iommu);

drivers/vfio/vfio_iommu_type1.c

2400

vfio_iommu_iova_insert_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2407

iommu->num_non_pinned_groups++;

drivers/vfio/vfio_iommu_type1.c

2408

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2424

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2428

static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2432

while ((node = rb_first(&iommu->dma_list)))

drivers/vfio/vfio_iommu_type1.c

2433

vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));

drivers/vfio/vfio_iommu_type1.c

2436

static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2440

n = rb_first(&iommu->dma_list);

drivers/vfio/vfio_iommu_type1.c

2446

unlocked += vfio_unmap_unpin(iommu, dma, false);

drivers/vfio/vfio_iommu_type1.c

2464

static void vfio_iommu_aper_expand(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2475

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2497

static int vfio_iommu_resv_refresh(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2510

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2541

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

2547

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2548

list_for_each_entry(group, &iommu->emulated_iommu_groups, next) {

drivers/vfio/vfio_iommu_type1.c

2555

if (list_empty(&iommu->emulated_iommu_groups) &&

drivers/vfio/vfio_iommu_type1.c

2556

list_empty(&iommu->domain_list)) {

drivers/vfio/vfio_iommu_type1.c

2557

WARN_ON(!list_empty(&iommu->device_list));

drivers/vfio/vfio_iommu_type1.c

2558

vfio_iommu_unmap_unpin_all(iommu);

drivers/vfio/vfio_iommu_type1.c

2568

vfio_iommu_iova_get_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2570

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2587

if (list_is_singular(&iommu->domain_list)) {

drivers/vfio/vfio_iommu_type1.c

2588

if (list_empty(&iommu->emulated_iommu_groups)) {

drivers/vfio/vfio_iommu_type1.c

2590

&iommu->device_list));

drivers/vfio/vfio_iommu_type1.c

2591

vfio_iommu_unmap_unpin_all(iommu);

drivers/vfio/vfio_iommu_type1.c

2593

vfio_iommu_unmap_unpin_reaccount(iommu);

drivers/vfio/vfio_iommu_type1.c

2599

vfio_iommu_aper_expand(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2600

vfio_update_pgsize_bitmap(iommu);

drivers/vfio/vfio_iommu_type1.c

2605

if (!vfio_iommu_resv_refresh(iommu, &iova_copy))

drivers/vfio/vfio_iommu_type1.c

2606

vfio_iommu_iova_insert_copy(iommu, &iova_copy);

drivers/vfio/vfio_iommu_type1.c

2616

iommu->num_non_pinned_groups--;

drivers/vfio/vfio_iommu_type1.c

2617

if (iommu->dirty_page_tracking)

drivers/vfio/vfio_iommu_type1.c

2618

vfio_iommu_populate_bitmap_full(iommu);

drivers/vfio/vfio_iommu_type1.c

2620

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2625

struct vfio_iommu *iommu;

drivers/vfio/vfio_iommu_type1.c

2627

iommu = kzalloc_obj(*iommu);

drivers/vfio/vfio_iommu_type1.c

2628

if (!iommu)

drivers/vfio/vfio_iommu_type1.c

2636

iommu->v2 = true;

drivers/vfio/vfio_iommu_type1.c

2639

kfree(iommu);

drivers/vfio/vfio_iommu_type1.c

2643

INIT_LIST_HEAD(&iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

2644

INIT_LIST_HEAD(&iommu->iova_list);

drivers/vfio/vfio_iommu_type1.c

2645

iommu->dma_list = RB_ROOT;

drivers/vfio/vfio_iommu_type1.c

2646

iommu->dma_avail = dma_entry_limit;

drivers/vfio/vfio_iommu_type1.c

2647

mutex_init(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2648

mutex_init(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

2649

INIT_LIST_HEAD(&iommu->device_list);

drivers/vfio/vfio_iommu_type1.c

2650

iommu->pgsize_bitmap = PAGE_MASK;

drivers/vfio/vfio_iommu_type1.c

2651

INIT_LIST_HEAD(&iommu->emulated_iommu_groups);

drivers/vfio/vfio_iommu_type1.c

2653

return iommu;

drivers/vfio/vfio_iommu_type1.c

2672

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

2677

&iommu->emulated_iommu_groups, next) {

drivers/vfio/vfio_iommu_type1.c

2682

vfio_iommu_unmap_unpin_all(iommu);

drivers/vfio/vfio_iommu_type1.c

2685

&iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2691

vfio_iommu_iova_free(&iommu->iova_list);

drivers/vfio/vfio_iommu_type1.c

2693

kfree(iommu);

drivers/vfio/vfio_iommu_type1.c

2696

static int vfio_domains_have_enforce_cache_coherency(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2701

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2702

list_for_each_entry(domain, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

2708

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2713

static bool vfio_iommu_has_emulated(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2717

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2718

ret = !list_empty(&iommu->emulated_iommu_groups);

drivers/vfio/vfio_iommu_type1.c

2719

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2723

static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2736

return iommu && !vfio_iommu_has_emulated(iommu);

drivers/vfio/vfio_iommu_type1.c

2738

if (!iommu)

drivers/vfio/vfio_iommu_type1.c

2740

return vfio_domains_have_enforce_cache_coherency(iommu);

drivers/vfio/vfio_iommu_type1.c

2767

static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2775

list_for_each_entry(iova, &iommu->iova_list, list)

drivers/vfio/vfio_iommu_type1.c

279

static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

2794

list_for_each_entry(iova, &iommu->iova_list, list) {

drivers/vfio/vfio_iommu_type1.c

2806

static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2816

cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

282

unsigned long pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

2822

static int vfio_iommu_dma_avail_build_caps(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2830

cap_dma_avail.avail = iommu->dma_avail;

drivers/vfio/vfio_iommu_type1.c

2836

static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

284

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

2854

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2857

info.iova_pgsizes = iommu->pgsize_bitmap;

drivers/vfio/vfio_iommu_type1.c

2859

ret = vfio_iommu_migration_build_caps(iommu, &caps);

drivers/vfio/vfio_iommu_type1.c

2862

ret = vfio_iommu_dma_avail_build_caps(iommu, &caps);

drivers/vfio/vfio_iommu_type1.c

2865

ret = vfio_iommu_iova_build_caps(iommu, &caps);

drivers/vfio/vfio_iommu_type1.c

2867

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2895

static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

291

static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize)

drivers/vfio/vfio_iommu_type1.c

2911

return vfio_dma_do_map(iommu, &map);

drivers/vfio/vfio_iommu_type1.c

2914

static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

295

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

2959

ret = vfio_dma_do_unmap(iommu, &unmap, &bitmap);

drivers/vfio/vfio_iommu_type1.c

2967

static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

2977

if (!iommu->v2)

drivers/vfio/vfio_iommu_type1.c

2995

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

2996

pgsize = 1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

2997

if (!iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

2998

ret = vfio_dma_bitmap_alloc_all(iommu, pgsize);

drivers/vfio/vfio_iommu_type1.c

3000

iommu->dirty_page_tracking = true;

drivers/vfio/vfio_iommu_type1.c

3002

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3005

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3006

if (iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

3007

iommu->dirty_page_tracking = false;

drivers/vfio/vfio_iommu_type1.c

3008

vfio_dma_bitmap_free_all(iommu);

drivers/vfio/vfio_iommu_type1.c

3010

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3048

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3050

iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

3066

if (iommu->dirty_page_tracking)

drivers/vfio/vfio_iommu_type1.c

3068

iommu, iova, iova_end,

drivers/vfio/vfio_iommu_type1.c

3073

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3084

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3088

return vfio_iommu_type1_check_extension(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3090

return vfio_iommu_type1_get_info(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3092

return vfio_iommu_type1_map_dma(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3094

return vfio_iommu_type1_unmap_dma(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3096

return vfio_iommu_type1_dirty_pages(iommu, arg);

drivers/vfio/vfio_iommu_type1.c

3105

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3116

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3117

mutex_lock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3118

list_add(&vdev->iommu_entry, &iommu->device_list);

drivers/vfio/vfio_iommu_type1.c

3119

mutex_unlock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3120

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3126

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3131

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3132

mutex_lock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3134

mutex_unlock(&iommu->device_list_lock);

drivers/vfio/vfio_iommu_type1.c

3135

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3138

static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu,

drivers/vfio/vfio_iommu_type1.c

3151

dma = vfio_find_dma(iommu, user_iova, 1);

drivers/vfio/vfio_iommu_type1.c

316

static void vfio_dma_bitmap_free_all(struct vfio_iommu *iommu)

drivers/vfio/vfio_iommu_type1.c

3178

if (*copied && iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

3179

unsigned long pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

320

for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {

drivers/vfio/vfio_iommu_type1.c

3201

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3205

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3207

if (WARN_ONCE(iommu->vaddr_invalid_count,

drivers/vfio/vfio_iommu_type1.c

3214

ret = vfio_iommu_type1_dma_rw_chunk(iommu, user_iova, data,

drivers/vfio/vfio_iommu_type1.c

3225

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3234

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

3237

if (!iommu || !iommu_group)

drivers/vfio/vfio_iommu_type1.c

3240

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

3241

list_for_each_entry(d, &iommu->domain_list, next) {

drivers/vfio/vfio_iommu_type1.c

3247

mutex_unlock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

899

struct vfio_iommu *iommu = iommu_data;

drivers/vfio/vfio_iommu_type1.c

908

if (!iommu || !pages || npage <= 0)

drivers/vfio/vfio_iommu_type1.c

912

if (!iommu->v2)

drivers/vfio/vfio_iommu_type1.c

919

mutex_lock(&iommu->lock);

drivers/vfio/vfio_iommu_type1.c

921

if (WARN_ONCE(iommu->vaddr_invalid_count,

drivers/vfio/vfio_iommu_type1.c

928

if (list_empty(&iommu->device_list)) {

drivers/vfio/vfio_iommu_type1.c

938

do_accounting = list_empty(&iommu->domain_list);

drivers/vfio/vfio_iommu_type1.c

946

dma = vfio_find_dma(iommu, iova, PAGE_SIZE);

drivers/vfio/vfio_iommu_type1.c

983

if (iommu->dirty_page_tracking) {

drivers/vfio/vfio_iommu_type1.c

984

unsigned long pgshift = __ffs(iommu->pgsize_bitmap);

drivers/vfio/vfio_iommu_type1.c

996

group = vfio_iommu_find_iommu_group(iommu, iommu_group);

drivers/vfio/vfio_iommu_type1.c

999

iommu->num_non_pinned_groups--;

drivers/xen/grant-dma-iommu.c

16

struct iommu_device iommu;

drivers/xen/grant-dma-iommu.c

45

ret = iommu_device_register(&mmu->iommu, &grant_dma_iommu_ops, &pdev->dev);

drivers/xen/grant-dma-iommu.c

59

iommu_device_unregister(&mmu->iommu);

include/linux/amd-iommu.h

65

int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn,

include/linux/amd-iommu.h

67

int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn,

include/linux/device.h

789

struct dev_iommu *iommu;

include/linux/dmar.h

130

void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,

include/linux/dmar.h

133

static inline void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,

include/linux/dmar.h

296

extern int dmar_set_interrupt(struct intel_iommu *iommu);

include/linux/dmar.h

49

struct intel_iommu *iommu;

include/linux/dmar.h

82

if (i=drhd->iommu, drhd->ignored) {} else

include/linux/dmar.h

87

if (i=drhd->iommu, 0) {} else

include/linux/generic_pt/iommu.h

264

struct pt_iommu iommu; \

include/linux/iommu.h

1182

if (dev->iommu)

include/linux/iommu.h

1183

return dev->iommu->fwspec;

include/linux/iommu.h

1191

dev->iommu->fwspec = fwspec;

include/linux/iommu.h

1196

if (dev->iommu)

include/linux/iommu.h

1197

return dev->iommu->priv;

include/linux/iommu.h

1419

static inline int iommu_device_register(struct iommu_device *iommu,

include/linux/iommu.h

1458

static inline void iommu_device_unregister(struct iommu_device *iommu)

include/linux/iommu.h

1462

static inline int iommu_device_sysfs_add(struct iommu_device *iommu,

include/linux/iommu.h

1470

static inline void iommu_device_sysfs_remove(struct iommu_device *iommu)

include/linux/iommu.h

890

int iommu_device_register(struct iommu_device *iommu,

include/linux/iommu.h

893

void iommu_device_unregister(struct iommu_device *iommu);

include/linux/iommu.h

894

int iommu_device_sysfs_add(struct iommu_device *iommu,

include/linux/iommu.h

898

void iommu_device_sysfs_remove(struct iommu_device *iommu);

include/linux/iommu.h

899

int iommu_device_link(struct iommu_device *iommu, struct device *link);

include/linux/iommu.h

900

void iommu_device_unlink(struct iommu_device *iommu, struct device *link);

include/linux/iommu.h

918

return dev->iommu->iommu_dev;

include/trace/events/iommu.h

9

#define TRACE_SYSTEM iommu

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

35

struct iommu *iommu_init(const char *iommu_mode);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

36

void iommu_cleanup(struct iommu *iommu);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

38

int __iommu_map(struct iommu *iommu, struct dma_region *region);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

40

static inline void iommu_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

42

VFIO_ASSERT_EQ(__iommu_map(iommu, region), 0);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

45

int __iommu_unmap(struct iommu *iommu, struct dma_region *region, u64 *unmapped);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

47

static inline void iommu_unmap(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

49

VFIO_ASSERT_EQ(__iommu_unmap(iommu, region, NULL), 0);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

52

int __iommu_unmap_all(struct iommu *iommu, u64 *unmapped);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

54

static inline void iommu_unmap_all(struct iommu *iommu)

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

56

VFIO_ASSERT_EQ(__iommu_unmap_all(iommu, NULL), 0);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

59

int __iommu_hva2iova(struct iommu *iommu, void *vaddr, iova_t *iova);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

60

iova_t iommu_hva2iova(struct iommu *iommu, void *vaddr);

tools/testing/selftests/vfio/lib/include/libvfio/iommu.h

62

struct iommu_iova_range *iommu_iova_ranges(struct iommu *iommu, u32 *nranges);

tools/testing/selftests/vfio/lib/include/libvfio/iova_allocator.h

18

struct iova_allocator *iova_allocator_init(struct iommu *iommu);

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

110

return __iommu_hva2iova(device->iommu, vaddr, iova);

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

115

return iommu_hva2iova(device->iommu, vaddr);

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

23

struct iommu *iommu;

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

41

struct vfio_pci_device *vfio_pci_device_alloc(const char *bdf, struct iommu *iommu);

tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h

43

struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu);

tools/testing/selftests/vfio/lib/iommu.c

100

static int vfio_iommu_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/iommu.c

110

if (ioctl(iommu->container_fd, VFIO_IOMMU_MAP_DMA, &args))

tools/testing/selftests/vfio/lib/iommu.c

116

static int iommufd_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/iommu.c

126

.ioas_id = iommu->ioas_id,

tools/testing/selftests/vfio/lib/iommu.c

129

if (ioctl(iommu->iommufd, IOMMU_IOAS_MAP, &args))

tools/testing/selftests/vfio/lib/iommu.c

135

int __iommu_map(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/lib/iommu.c

139

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

140

ret = iommufd_map(iommu, region);

tools/testing/selftests/vfio/lib/iommu.c

142

ret = vfio_iommu_map(iommu, region);

tools/testing/selftests/vfio/lib/iommu.c

147

list_add(&region->link, &iommu->dma_regions);

tools/testing/selftests/vfio/lib/iommu.c

170

static int vfio_iommu_unmap(struct iommu *iommu, struct dma_region *region,

tools/testing/selftests/vfio/lib/iommu.c

173

return __vfio_iommu_unmap(iommu->container_fd, region->iova,

tools/testing/selftests/vfio/lib/iommu.c

195

static int iommufd_unmap(struct iommu *iommu, struct dma_region *region,

tools/testing/selftests/vfio/lib/iommu.c

198

return __iommufd_unmap(iommu->iommufd, region->iova, region->size,

tools/testing/selftests/vfio/lib/iommu.c

199

iommu->ioas_id, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

202

int __iommu_unmap(struct iommu *iommu, struct dma_region *region, u64 *unmapped)

tools/testing/selftests/vfio/lib/iommu.c

206

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

207

ret = iommufd_unmap(iommu, region, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

209

ret = vfio_iommu_unmap(iommu, region, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

219

int __iommu_unmap_all(struct iommu *iommu, u64 *unmapped)

tools/testing/selftests/vfio/lib/iommu.c

224

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

225

ret = __iommufd_unmap(iommu->iommufd, 0, UINT64_MAX,

tools/testing/selftests/vfio/lib/iommu.c

226

iommu->ioas_id, unmapped);

tools/testing/selftests/vfio/lib/iommu.c

228

ret = __vfio_iommu_unmap(iommu->container_fd, 0, 0,

tools/testing/selftests/vfio/lib/iommu.c

234

list_for_each_entry_safe(curr, next, &iommu->dma_regions, link)

tools/testing/selftests/vfio/lib/iommu.c

312

static struct iommu_iova_range *vfio_iommu_iova_ranges(struct iommu *iommu,

tools/testing/selftests/vfio/lib/iommu.c

320

info = vfio_iommu_get_info(iommu->container_fd);

tools/testing/selftests/vfio/lib/iommu.c

344

static struct iommu_iova_range *iommufd_iova_ranges(struct iommu *iommu,

tools/testing/selftests/vfio/lib/iommu.c

352

.ioas_id = iommu->ioas_id,

tools/testing/selftests/vfio/lib/iommu.c

355

ret = ioctl(iommu->iommufd, IOMMU_IOAS_IOVA_RANGES, &query);

tools/testing/selftests/vfio/lib/iommu.c

365

ioctl_assert(iommu->iommufd, IOMMU_IOAS_IOVA_RANGES, &query);

tools/testing/selftests/vfio/lib/iommu.c

385

struct iommu_iova_range *iommu_iova_ranges(struct iommu *iommu, u32 *nranges)

tools/testing/selftests/vfio/lib/iommu.c

389

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

390

ranges = iommufd_iova_ranges(iommu, nranges);

tools/testing/selftests/vfio/lib/iommu.c

392

ranges = vfio_iommu_iova_ranges(iommu, nranges);

tools/testing/selftests/vfio/lib/iommu.c

421

struct iommu *iommu_init(const char *iommu_mode)

tools/testing/selftests/vfio/lib/iommu.c

424

struct iommu *iommu;

tools/testing/selftests/vfio/lib/iommu.c

427

iommu = calloc(1, sizeof(*iommu));

tools/testing/selftests/vfio/lib/iommu.c

428

VFIO_ASSERT_NOT_NULL(iommu);

tools/testing/selftests/vfio/lib/iommu.c

430

INIT_LIST_HEAD(&iommu->dma_regions);

tools/testing/selftests/vfio/lib/iommu.c

432

iommu->mode = lookup_iommu_mode(iommu_mode);

tools/testing/selftests/vfio/lib/iommu.c

434

container_path = iommu->mode->container_path;

tools/testing/selftests/vfio/lib/iommu.c

436

iommu->container_fd = open(container_path, O_RDWR);

tools/testing/selftests/vfio/lib/iommu.c

437

VFIO_ASSERT_GE(iommu->container_fd, 0, "open(%s) failed\n", container_path);

tools/testing/selftests/vfio/lib/iommu.c

439

version = ioctl(iommu->container_fd, VFIO_GET_API_VERSION);

tools/testing/selftests/vfio/lib/iommu.c

447

iommu->iommufd = open("/dev/iommu", O_RDWR);

tools/testing/selftests/vfio/lib/iommu.c

448

VFIO_ASSERT_GT(iommu->iommufd, 0);

tools/testing/selftests/vfio/lib/iommu.c

450

iommu->ioas_id = iommufd_ioas_alloc(iommu->iommufd);

tools/testing/selftests/vfio/lib/iommu.c

453

return iommu;

tools/testing/selftests/vfio/lib/iommu.c

456

void iommu_cleanup(struct iommu *iommu)

tools/testing/selftests/vfio/lib/iommu.c

458

if (iommu->iommufd)

tools/testing/selftests/vfio/lib/iommu.c

459

VFIO_ASSERT_EQ(close(iommu->iommufd), 0);

tools/testing/selftests/vfio/lib/iommu.c

461

VFIO_ASSERT_EQ(close(iommu->container_fd), 0);

tools/testing/selftests/vfio/lib/iommu.c

463

free(iommu);

tools/testing/selftests/vfio/lib/iommu.c

69

int __iommu_hva2iova(struct iommu *iommu, void *vaddr, iova_t *iova)

tools/testing/selftests/vfio/lib/iommu.c

73

list_for_each_entry(region, &iommu->dma_regions, link) {

tools/testing/selftests/vfio/lib/iommu.c

89

iova_t iommu_hva2iova(struct iommu *iommu, void *vaddr)

tools/testing/selftests/vfio/lib/iommu.c

94

ret = __iommu_hva2iova(iommu, vaddr, &iova);

tools/testing/selftests/vfio/lib/iova_allocator.c

23

struct iova_allocator *iova_allocator_init(struct iommu *iommu)

tools/testing/selftests/vfio/lib/iova_allocator.c

29

ranges = iommu_iova_ranges(iommu, &nranges);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

258

ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->iommu->container_fd);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

288

struct iommu *iommu = device->iommu;

tools/testing/selftests/vfio/lib/vfio_pci_device.c

289

unsigned long iommu_type = iommu->mode->iommu_type;

tools/testing/selftests/vfio/lib/vfio_pci_device.c

292

ret = ioctl(iommu->container_fd, VFIO_CHECK_EXTENSION, iommu_type);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

300

(void)ioctl(iommu->container_fd, VFIO_SET_IOMMU, (void *)iommu_type);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

420

vfio_device_bind_iommufd(device->fd, device->iommu->iommufd, vf_token);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

421

vfio_device_attach_iommufd_pt(device->fd, device->iommu->ioas_id);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

424

struct vfio_pci_device *vfio_pci_device_alloc(const char *bdf, struct iommu *iommu)

tools/testing/selftests/vfio/lib/vfio_pci_device.c

431

VFIO_ASSERT_NOT_NULL(iommu);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

432

device->iommu = iommu;

tools/testing/selftests/vfio/lib/vfio_pci_device.c

443

struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu)

tools/testing/selftests/vfio/lib/vfio_pci_device.c

447

device = vfio_pci_device_alloc(bdf, iommu);

tools/testing/selftests/vfio/lib/vfio_pci_device.c

449

if (iommu->mode->container_path)

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

100

iommu_unmap(iommu, &region);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

102

VFIO_ASSERT_NE(__iommu_map(iommu, &region), 0);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

108

do_mmio_map_test(self->iommu, self->iova_allocator,

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

117

do_mmio_map_test(self->iommu, self->iova_allocator,

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

133

do_mmio_map_test(self->iommu, self->iova_allocator, vaddr, size);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

46

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

67

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

68

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

69

self->iova_allocator = iova_allocator_init(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

80

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

83

static void do_mmio_map_test(struct iommu *iommu,

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

97

if (!strcmp(iommu->mode->name, MODE_VFIO_TYPE1V2_IOMMU) ||

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

98

!strcmp(iommu->mode->name, MODE_VFIO_TYPE1_IOMMU)) {

tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c

99

iommu_map(iommu, &region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

122

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

123

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

124

self->iova_allocator = iova_allocator_init(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

131

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

155

iommu_map(self->iommu, &region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

164

if (self->iommu->mode->iommu_type == VFIO_TYPE1_IOMMU)

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

193

rc = __iommu_unmap(self->iommu, &region, &unmapped);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

204

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

237

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

238

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

243

ranges = iommu_iova_ranges(self->iommu, &nranges);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

256

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

266

iommu_map(self->iommu, region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

269

rc = __iommu_unmap(self->iommu, region, &unmapped);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

280

iommu_map(self->iommu, region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

283

rc = __iommu_unmap_all(self->iommu, &unmapped);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

296

rc = __iommu_map(self->iommu, region);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

299

rc = __iommu_unmap(self->iommu, region, NULL);

tools/testing/selftests/vfio/vfio_dma_mapping_test.c

96

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

105

device = vfio_pci_device_init(device_bdfs[args->device_index], args->iommu);

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

18

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

29

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

47

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

54

self->thread_args[i].iommu = self->iommu;

tools/testing/selftests/vfio/vfio_pci_device_init_perf_test.c

62

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

106

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_test.c

124

self->iommu = iommu_init(default_iommu_mode);

tools/testing/selftests/vfio/vfio_pci_device_test.c

125

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

131

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

26

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_device_test.c

32

self->iommu = iommu_init(default_iommu_mode);

tools/testing/selftests/vfio/vfio_pci_device_test.c

33

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_pci_device_test.c

39

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

112

region_teardown(self->iommu, &self->memcpy_region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

113

region_teardown(self->iommu, &driver->region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

117

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

21

static void region_setup(struct iommu *iommu,

tools/testing/selftests/vfio/vfio_pci_driver_test.c

239

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_driver_test.c

242

iommu = iommu_init(default_iommu_mode);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

243

device = vfio_pci_device_init(device_bdf, iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

248

iommu_cleanup(iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

36

iommu_map(iommu, region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

39

static void region_teardown(struct iommu *iommu, struct dma_region *region)

tools/testing/selftests/vfio/vfio_pci_driver_test.c

41

iommu_unmap(iommu, region);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

46

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_driver_test.c

76

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

77

self->device = vfio_pci_device_init(device_bdf, self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

78

self->iova_allocator = iova_allocator_init(self->iommu);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

82

region_setup(self->iommu, self->iova_allocator, &self->memcpy_region, SZ_1G);

tools/testing/selftests/vfio/vfio_pci_driver_test.c

83

region_setup(self->iommu, self->iova_allocator, &driver->region, SZ_2M);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

103

iommu_cleanup(self->iommu);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

125

ret = device_init(vf_bdf, self->iommu, variant->vf_token, &self->vf);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

141

ret = device_init(vf_bdf, self->iommu, variant->vf_token, &self->vf);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

156

ret = device_init(vf_bdf, self->iommu, variant->vf_token, &self->vf);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

37

device->iommu->iommufd, vf_token);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

40

static int device_init(const char *bdf, struct iommu *iommu,

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

43

struct vfio_pci_device *device = vfio_pci_device_alloc(bdf, iommu);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

46

if (iommu->mode->container_path)

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

72

struct iommu *iommu;

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

93

self->iommu = iommu_init(variant->iommu_mode);

tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c

96

ASSERT_EQ(device_init(pf_bdf, self->iommu, self->pf_token, &self->pf), 0);