#ifndef __SELFTEST_IOMMUFD_UTILS
#define __SELFTEST_IOMMUFD_UTILS
#include <unistd.h>
#include <stddef.h>
#include <sys/fcntl.h>
#include <sys/ioctl.h>
#include <stdint.h>
#include <assert.h>
#include <poll.h>
#include "kselftest_harness.h"
#include "../../../../drivers/iommu/iommufd/iommufd_test.h"
#define _IOMMU_TEST_CMD(x) IOMMU_TEST_CMD
#define BITS_PER_BYTE 8
#define BITS_PER_LONG __BITS_PER_LONG
#define BIT_MASK(nr) (1UL << ((nr) % __BITS_PER_LONG))
#define BIT_WORD(nr) ((nr) / __BITS_PER_LONG)
enum {
IOPT_PAGES_ACCOUNT_NONE = 0,
IOPT_PAGES_ACCOUNT_USER = 1,
IOPT_PAGES_ACCOUNT_MM = 2,
};
#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
static inline void set_bit(unsigned int nr, unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
*p |= mask;
}
static inline bool test_bit(unsigned int nr, unsigned long *addr)
{
return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)));
}
static void *buffer;
static unsigned long BUFFER_SIZE;
static void *mfd_buffer;
static int mfd;
static unsigned long PAGE_SIZE;
#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
#define offsetofend(TYPE, MEMBER) \
(offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
#define test_err_mmap(_errno, length, offset) \
EXPECT_ERRNO(_errno, (long)mmap(NULL, length, PROT_READ | PROT_WRITE, \
MAP_SHARED, self->fd, offset))
static inline void *memfd_mmap(size_t length, int prot, int flags, int *mfd_p)
{
int mfd_flags = (flags & MAP_HUGETLB) ? MFD_HUGETLB : 0;
int mfd = memfd_create("buffer", mfd_flags);
void *buf = MAP_FAILED;
if (mfd <= 0)
return MAP_FAILED;
if (ftruncate(mfd, length))
goto out;
*mfd_p = mfd;
buf = mmap(0, length, prot, flags, mfd, 0);
out:
if (buf == MAP_FAILED)
close(mfd);
return buf;
}
#define check_refs(_ptr, _length, _refs) \
({ \
struct iommu_test_cmd test_cmd = { \
.size = sizeof(test_cmd), \
.op = IOMMU_TEST_OP_MD_CHECK_REFS, \
.check_refs = { .length = _length, \
.uptr = (uintptr_t)(_ptr), \
.refs = _refs }, \
}; \
ASSERT_EQ(0, \
ioctl(self->fd, \
_IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS), \
&test_cmd)); \
})
static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *stdev_id,
__u32 *hwpt_id, __u32 *idev_id)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_MOCK_DOMAIN,
.id = ioas_id,
.mock_domain = {},
};
int ret;
ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
if (ret)
return ret;
if (stdev_id)
*stdev_id = cmd.mock_domain.out_stdev_id;
assert(cmd.id != 0);
if (hwpt_id)
*hwpt_id = cmd.mock_domain.out_hwpt_id;
if (idev_id)
*idev_id = cmd.mock_domain.out_idev_id;
return 0;
}
#define test_cmd_mock_domain(ioas_id, stdev_id, hwpt_id, idev_id) \
ASSERT_EQ(0, _test_cmd_mock_domain(self->fd, ioas_id, stdev_id, \
hwpt_id, idev_id))
#define test_err_mock_domain(_errno, ioas_id, stdev_id, hwpt_id) \
EXPECT_ERRNO(_errno, _test_cmd_mock_domain(self->fd, ioas_id, \
stdev_id, hwpt_id, NULL))
static int _test_cmd_mock_domain_flags(int fd, unsigned int ioas_id,
__u32 stdev_flags, __u32 *stdev_id,
__u32 *hwpt_id, __u32 *idev_id)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS,
.id = ioas_id,
.mock_domain_flags = { .dev_flags = stdev_flags },
};
int ret;
ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
if (ret)
return ret;
if (stdev_id)
*stdev_id = cmd.mock_domain_flags.out_stdev_id;
assert(cmd.id != 0);
if (hwpt_id)
*hwpt_id = cmd.mock_domain_flags.out_hwpt_id;
if (idev_id)
*idev_id = cmd.mock_domain_flags.out_idev_id;
return 0;
}
#define test_cmd_mock_domain_flags(ioas_id, flags, stdev_id, hwpt_id, idev_id) \
ASSERT_EQ(0, _test_cmd_mock_domain_flags(self->fd, ioas_id, flags, \
stdev_id, hwpt_id, idev_id))
#define test_err_mock_domain_flags(_errno, ioas_id, flags, stdev_id, hwpt_id) \
EXPECT_ERRNO(_errno, \
_test_cmd_mock_domain_flags(self->fd, ioas_id, flags, \
stdev_id, hwpt_id, NULL))
static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id,
__u32 *hwpt_id)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE,
.id = stdev_id,
.mock_domain_replace = {
.pt_id = pt_id,
},
};
int ret;
ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
if (ret)
return ret;
if (hwpt_id)
*hwpt_id = cmd.mock_domain_replace.pt_id;
return 0;
}
#define test_cmd_mock_domain_replace(stdev_id, pt_id) \
ASSERT_EQ(0, _test_cmd_mock_domain_replace(self->fd, stdev_id, pt_id, \
NULL))
#define test_err_mock_domain_replace(_errno, stdev_id, pt_id) \
EXPECT_ERRNO(_errno, _test_cmd_mock_domain_replace(self->fd, stdev_id, \
pt_id, NULL))
static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, __u32 ft_id,
__u32 flags, __u32 *hwpt_id, __u32 data_type,
void *data, size_t data_len)
{
struct iommu_hwpt_alloc cmd = {
.size = sizeof(cmd),
.flags = flags,
.dev_id = device_id,
.pt_id = pt_id,
.data_type = data_type,
.data_len = data_len,
.data_uptr = (uint64_t)data,
.fault_id = ft_id,
};
int ret;
ret = ioctl(fd, IOMMU_HWPT_ALLOC, &cmd);
if (ret)
return ret;
if (hwpt_id)
*hwpt_id = cmd.out_hwpt_id;
return 0;
}
#define test_cmd_hwpt_alloc(device_id, pt_id, flags, hwpt_id) \
ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, \
0))
#define test_cmd_hwpt_alloc_iommupt(device_id, pt_id, flags, iommupt_type, \
hwpt_id) \
({ \
struct iommu_hwpt_selftest user_cfg = { \
.pagetable_type = iommupt_type \
}; \
\
ASSERT_EQ(0, _test_cmd_hwpt_alloc( \
self->fd, device_id, pt_id, 0, flags, \
hwpt_id, IOMMU_HWPT_DATA_SELFTEST, \
&user_cfg, sizeof(user_cfg))); \
})
#define test_err_hwpt_alloc(_errno, device_id, pt_id, flags, hwpt_id) \
EXPECT_ERRNO(_errno, _test_cmd_hwpt_alloc( \
self->fd, device_id, pt_id, 0, flags, \
hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, 0))
#define test_cmd_hwpt_alloc_nested(device_id, pt_id, flags, hwpt_id, \
data_type, data, data_len) \
ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
hwpt_id, data_type, data, data_len))
#define test_err_hwpt_alloc_nested(_errno, device_id, pt_id, flags, hwpt_id, \
data_type, data, data_len) \
EXPECT_ERRNO(_errno, \
_test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
hwpt_id, data_type, data, data_len))
#define test_cmd_hwpt_alloc_iopf(device_id, pt_id, fault_id, flags, hwpt_id, \
data_type, data, data_len) \
ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
flags, hwpt_id, data_type, data, \
data_len))
#define test_err_hwpt_alloc_iopf(_errno, device_id, pt_id, fault_id, flags, \
hwpt_id, data_type, data, data_len) \
EXPECT_ERRNO(_errno, \
_test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
flags, hwpt_id, data_type, data, \
data_len))
#define test_cmd_hwpt_check_iotlb(hwpt_id, iotlb_id, expected) \
({ \
struct iommu_test_cmd test_cmd = { \
.size = sizeof(test_cmd), \
.op = IOMMU_TEST_OP_MD_CHECK_IOTLB, \
.id = hwpt_id, \
.check_iotlb = { \
.id = iotlb_id, \
.iotlb = expected, \
}, \
}; \
ASSERT_EQ(0, \
ioctl(self->fd, \
_IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_IOTLB), \
&test_cmd)); \
})
#define test_cmd_hwpt_check_iotlb_all(hwpt_id, expected) \
({ \
int i; \
for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++) \
test_cmd_hwpt_check_iotlb(hwpt_id, i, expected); \
})
#define test_cmd_dev_check_cache(device_id, cache_id, expected) \
({ \
struct iommu_test_cmd test_cmd = { \
.size = sizeof(test_cmd), \
.op = IOMMU_TEST_OP_DEV_CHECK_CACHE, \
.id = device_id, \
.check_dev_cache = { \
.id = cache_id, \
.cache = expected, \
}, \
}; \
ASSERT_EQ(0, ioctl(self->fd, \
_IOMMU_TEST_CMD( \
IOMMU_TEST_OP_DEV_CHECK_CACHE), \
&test_cmd)); \
})
#define test_cmd_dev_check_cache_all(device_id, expected) \
({ \
int c; \
for (c = 0; c < MOCK_DEV_CACHE_NUM; c++) \
test_cmd_dev_check_cache(device_id, c, expected); \
})
static int _test_cmd_hwpt_invalidate(int fd, __u32 hwpt_id, void *reqs,
uint32_t data_type, uint32_t lreq,
uint32_t *nreqs)
{
struct iommu_hwpt_invalidate cmd = {
.size = sizeof(cmd),
.hwpt_id = hwpt_id,
.data_type = data_type,
.data_uptr = (uint64_t)reqs,
.entry_len = lreq,
.entry_num = *nreqs,
};
int rc = ioctl(fd, IOMMU_HWPT_INVALIDATE, &cmd);
*nreqs = cmd.entry_num;
return rc;
}
#define test_cmd_hwpt_invalidate(hwpt_id, reqs, data_type, lreq, nreqs) \
({ \
ASSERT_EQ(0, \
_test_cmd_hwpt_invalidate(self->fd, hwpt_id, reqs, \
data_type, lreq, nreqs)); \
})
#define test_err_hwpt_invalidate(_errno, hwpt_id, reqs, data_type, lreq, \
nreqs) \
({ \
EXPECT_ERRNO(_errno, _test_cmd_hwpt_invalidate( \
self->fd, hwpt_id, reqs, \
data_type, lreq, nreqs)); \
})
static int _test_cmd_viommu_invalidate(int fd, __u32 viommu_id, void *reqs,
uint32_t data_type, uint32_t lreq,
uint32_t *nreqs)
{
struct iommu_hwpt_invalidate cmd = {
.size = sizeof(cmd),
.hwpt_id = viommu_id,
.data_type = data_type,
.data_uptr = (uint64_t)reqs,
.entry_len = lreq,
.entry_num = *nreqs,
};
int rc = ioctl(fd, IOMMU_HWPT_INVALIDATE, &cmd);
*nreqs = cmd.entry_num;
return rc;
}
#define test_cmd_viommu_invalidate(viommu, reqs, lreq, nreqs) \
({ \
ASSERT_EQ(0, \
_test_cmd_viommu_invalidate(self->fd, viommu, reqs, \
IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, \
lreq, nreqs)); \
})
#define test_err_viommu_invalidate(_errno, viommu_id, reqs, data_type, lreq, \
nreqs) \
({ \
EXPECT_ERRNO(_errno, _test_cmd_viommu_invalidate( \
self->fd, viommu_id, reqs, \
data_type, lreq, nreqs)); \
})
static int _test_cmd_access_replace_ioas(int fd, __u32 access_id,
unsigned int ioas_id)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_ACCESS_REPLACE_IOAS,
.id = access_id,
.access_replace_ioas = { .ioas_id = ioas_id },
};
int ret;
ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
if (ret)
return ret;
return 0;
}
#define test_cmd_access_replace_ioas(access_id, ioas_id) \
ASSERT_EQ(0, _test_cmd_access_replace_ioas(self->fd, access_id, ioas_id))
static int _test_cmd_set_dirty_tracking(int fd, __u32 hwpt_id, bool enabled)
{
struct iommu_hwpt_set_dirty_tracking cmd = {
.size = sizeof(cmd),
.flags = enabled ? IOMMU_HWPT_DIRTY_TRACKING_ENABLE : 0,
.hwpt_id = hwpt_id,
};
int ret;
ret = ioctl(fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &cmd);
if (ret)
return -errno;
return 0;
}
#define test_cmd_set_dirty_tracking(hwpt_id, enabled) \
ASSERT_EQ(0, _test_cmd_set_dirty_tracking(self->fd, hwpt_id, enabled))
static int _test_cmd_get_dirty_bitmap(int fd, __u32 hwpt_id, size_t length,
__u64 iova, size_t page_size,
__u64 *bitmap, __u32 flags)
{
struct iommu_hwpt_get_dirty_bitmap cmd = {
.size = sizeof(cmd),
.hwpt_id = hwpt_id,
.flags = flags,
.iova = iova,
.length = length,
.page_size = page_size,
.data = (uintptr_t)bitmap,
};
int ret;
ret = ioctl(fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &cmd);
if (ret)
return ret;
return 0;
}
#define test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size, \
bitmap, flags) \
ASSERT_EQ(0, _test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, \
page_size, bitmap, flags))
static int _test_cmd_mock_domain_set_dirty(int fd, __u32 hwpt_id, size_t length,
__u64 iova, size_t page_size,
__u64 *bitmap, __u64 *dirty)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_DIRTY,
.id = hwpt_id,
.dirty = {
.iova = iova,
.length = length,
.page_size = page_size,
.uptr = (uintptr_t)bitmap,
}
};
int ret;
ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_DIRTY), &cmd);
if (ret)
return -ret;
if (dirty)
*dirty = cmd.dirty.out_nr_dirty;
return 0;
}
#define test_cmd_mock_domain_set_dirty(fd, hwpt_id, length, iova, page_size, \
bitmap, nr) \
ASSERT_EQ(0, \
_test_cmd_mock_domain_set_dirty(fd, hwpt_id, length, iova, \
page_size, bitmap, nr))
static int _test_mock_dirty_bitmaps(int fd, __u32 hwpt_id, size_t length,
__u64 iova, size_t page_size,
size_t pte_page_size, __u64 *bitmap,
__u64 nbits, __u32 flags,
struct __test_metadata *_metadata)
{
unsigned long npte = pte_page_size / page_size, pteset = 2 * npte;
unsigned long j, i, nr = nbits / pteset ?: 1;
unsigned long bitmap_size = DIV_ROUND_UP(nbits, BITS_PER_BYTE);
__u64 out_dirty = 0;
memset(bitmap, 0, bitmap_size);
for (i = 0; i < nbits; i += pteset)
set_bit(i, (unsigned long *)bitmap);
test_cmd_mock_domain_set_dirty(fd, hwpt_id, length, iova, page_size,
bitmap, &out_dirty);
ASSERT_EQ(nr, out_dirty);
memset(bitmap, 0, bitmap_size);
test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size, bitmap,
flags);
for (i = 0; i < nbits; i += pteset) {
for (j = 0; j < pteset; j++) {
ASSERT_EQ(j < npte,
test_bit(i + j, (unsigned long *)bitmap));
}
ASSERT_EQ(!(i % pteset), test_bit(i, (unsigned long *)bitmap));
}
memset(bitmap, 0, bitmap_size);
test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size, bitmap,
flags);
for (i = 0; i < nbits; i += pteset) {
for (j = 0; j < pteset; j++) {
ASSERT_EQ(
(j < npte) &&
(flags &
IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR),
test_bit(i + j, (unsigned long *)bitmap));
}
}
return 0;
}
#define test_mock_dirty_bitmaps(hwpt_id, length, iova, page_size, pte_size,\
bitmap, bitmap_size, flags, _metadata) \
ASSERT_EQ(0, _test_mock_dirty_bitmaps(self->fd, hwpt_id, length, iova, \
page_size, pte_size, bitmap, \
bitmap_size, flags, _metadata))
static int _test_cmd_create_access(int fd, unsigned int ioas_id,
__u32 *access_id, unsigned int flags)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_CREATE_ACCESS,
.id = ioas_id,
.create_access = { .flags = flags },
};
int ret;
ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
if (ret)
return ret;
*access_id = cmd.create_access.out_access_fd;
return 0;
}
#define test_cmd_create_access(ioas_id, access_id, flags) \
ASSERT_EQ(0, _test_cmd_create_access(self->fd, ioas_id, access_id, \
flags))
static int _test_cmd_destroy_access(unsigned int access_id)
{
return close(access_id);
}
#define test_cmd_destroy_access(access_id) \
ASSERT_EQ(0, _test_cmd_destroy_access(access_id))
static int _test_cmd_destroy_access_pages(int fd, unsigned int access_id,
unsigned int access_pages_id)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES,
.id = access_id,
.destroy_access_pages = { .access_pages_id = access_pages_id },
};
return ioctl(fd, IOMMU_TEST_CMD, &cmd);
}
#define test_cmd_destroy_access_pages(access_id, access_pages_id) \
ASSERT_EQ(0, _test_cmd_destroy_access_pages(self->fd, access_id, \
access_pages_id))
#define test_err_destroy_access_pages(_errno, access_id, access_pages_id) \
EXPECT_ERRNO(_errno, _test_cmd_destroy_access_pages( \
self->fd, access_id, access_pages_id))
static int _test_cmd_get_dmabuf(int fd, size_t len, int *out_fd)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_DMABUF_GET,
.dmabuf_get = { .length = len, .open_flags = O_CLOEXEC },
};
*out_fd = ioctl(fd, IOMMU_TEST_CMD, &cmd);
if (*out_fd < 0)
return -1;
return 0;
}
#define test_cmd_get_dmabuf(len, out_fd) \
ASSERT_EQ(0, _test_cmd_get_dmabuf(self->fd, len, out_fd))
static int _test_cmd_revoke_dmabuf(int fd, int dmabuf_fd, bool revoked)
{
struct iommu_test_cmd cmd = {
.size = sizeof(cmd),
.op = IOMMU_TEST_OP_DMABUF_REVOKE,
.dmabuf_revoke = { .dmabuf_fd = dmabuf_fd, .revoked = revoked },
};
int ret;
ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
if (ret < 0)
return -1;
return 0;
}
#define test_cmd_revoke_dmabuf(dmabuf_fd, revoke) \
ASSERT_EQ(0, _test_cmd_revoke_dmabuf(self->fd, dmabuf_fd, revoke))
static int _test_ioctl_destroy(int fd, unsigned int id)
{
struct iommu_destroy cmd = {
.size = sizeof(cmd),
.id = id,
};
return ioctl(fd, IOMMU_DESTROY, &cmd);
}
#define test_ioctl_destroy(id) ASSERT_EQ(0, _test_ioctl_destroy(self->fd, id))
static int _test_ioctl_ioas_alloc(int fd, __u32 *id)
{
struct iommu_ioas_alloc cmd = {
.size = sizeof(cmd),
};
int ret;
ret = ioctl(fd, IOMMU_IOAS_ALLOC, &cmd);
if (ret)
return ret;
*id = cmd.out_ioas_id;
return 0;
}
#define test_ioctl_ioas_alloc(id) \
({ \
ASSERT_EQ(0, _test_ioctl_ioas_alloc(self->fd, id)); \
ASSERT_NE(0, *(id)); \
})
static int _test_ioctl_ioas_map(int fd, unsigned int ioas_id, void *buffer,
size_t length, __u64 *iova, unsigned int flags)
{
struct iommu_ioas_map cmd = {
.size = sizeof(cmd),
.flags = flags,
.ioas_id = ioas_id,
.user_va = (uintptr_t)buffer,
.length = length,
};
int ret;
if (flags & IOMMU_IOAS_MAP_FIXED_IOVA)
cmd.iova = *iova;
ret = ioctl(fd, IOMMU_IOAS_MAP, &cmd);
*iova = cmd.iova;
return ret;
}
#define test_ioctl_ioas_map(buffer, length, iova_p) \
ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
length, iova_p, \
IOMMU_IOAS_MAP_WRITEABLE | \
IOMMU_IOAS_MAP_READABLE))
#define test_err_ioctl_ioas_map(_errno, buffer, length, iova_p) \
EXPECT_ERRNO(_errno, \
_test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
length, iova_p, \
IOMMU_IOAS_MAP_WRITEABLE | \
IOMMU_IOAS_MAP_READABLE))
#define test_ioctl_ioas_map_id(ioas_id, buffer, length, iova_p) \
ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, ioas_id, buffer, length, \
iova_p, \
IOMMU_IOAS_MAP_WRITEABLE | \
IOMMU_IOAS_MAP_READABLE))
#define test_ioctl_ioas_map_fixed(buffer, length, iova) \
({ \
__u64 __iova = iova; \
ASSERT_EQ(0, _test_ioctl_ioas_map( \
self->fd, self->ioas_id, buffer, length, \
&__iova, \
IOMMU_IOAS_MAP_FIXED_IOVA | \
IOMMU_IOAS_MAP_WRITEABLE | \
IOMMU_IOAS_MAP_READABLE)); \
})
#define test_ioctl_ioas_map_fixed_id(ioas_id, buffer, length, iova) \
({ \
__u64 __iova = iova; \
ASSERT_EQ(0, \
_test_ioctl_ioas_map( \
self->fd, ioas_id, buffer, length, &__iova, \
IOMMU_IOAS_MAP_FIXED_IOVA | \
IOMMU_IOAS_MAP_WRITEABLE | \
IOMMU_IOAS_MAP_READABLE)); \
})
#define test_err_ioctl_ioas_map_fixed(_errno, buffer, length, iova) \
({ \
__u64 __iova = iova; \
EXPECT_ERRNO(_errno, \
_test_ioctl_ioas_map( \
self->fd, self->ioas_id, buffer, length, \
&__iova, \
IOMMU_IOAS_MAP_FIXED_IOVA | \
IOMMU_IOAS_MAP_WRITEABLE | \
IOMMU_IOAS_MAP_READABLE)); \
})
static int _test_ioctl_ioas_unmap(int fd, unsigned int ioas_id, uint64_t iova,
size_t length, uint64_t *out_len)
{
struct iommu_ioas_unmap cmd = {
.size = sizeof(cmd),
.ioas_id = ioas_id,
.iova = iova,
.length = length,
};
int ret;
ret = ioctl(fd, IOMMU_IOAS_UNMAP, &cmd);
if (out_len)
*out_len = cmd.length;
return ret;
}
#define test_ioctl_ioas_unmap(iova, length) \
ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, iova, \
length, NULL))
#define test_ioctl_ioas_unmap_id(ioas_id, iova, length) \
ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, ioas_id, iova, length, \
NULL))
#define test_err_ioctl_ioas_unmap(_errno, iova, length) \
EXPECT_ERRNO(_errno, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, \
iova, length, NULL))
static int _test_ioctl_ioas_map_file(int fd, unsigned int ioas_id, int mfd,
size_t start, size_t length, __u64 *iova,
unsigned int flags)
{
struct iommu_ioas_map_file cmd = {
.size = sizeof(cmd),
.flags = flags,
.ioas_id = ioas_id,
.fd = mfd,
.start = start,
.length = length,
};
int ret;
if (flags & IOMMU_IOAS_MAP_FIXED_IOVA)
cmd.iova = *iova;
ret = ioctl(fd, IOMMU_IOAS_MAP_FILE, &cmd);
*iova = cmd.iova;
return ret;
}
#define test_ioctl_ioas_map_file(mfd, start, length, iova_p) \
ASSERT_EQ(0, \
_test_ioctl_ioas_map_file( \
self->fd, self->ioas_id, mfd, start, length, iova_p, \
IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))
#define test_err_ioctl_ioas_map_file(_errno, mfd, start, length, iova_p) \
EXPECT_ERRNO( \
_errno, \
_test_ioctl_ioas_map_file( \
self->fd, self->ioas_id, mfd, start, length, iova_p, \
IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))
#define test_ioctl_ioas_map_id_file(ioas_id, mfd, start, length, iova_p) \
ASSERT_EQ(0, \
_test_ioctl_ioas_map_file( \
self->fd, ioas_id, mfd, start, length, iova_p, \
IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))
#define test_ioctl_ioas_map_fixed_file(mfd, start, length, iova) \
({ \
__u64 __iova = iova; \
ASSERT_EQ(0, _test_ioctl_ioas_map_file( \
self->fd, self->ioas_id, mfd, start, \
length, &__iova, \
IOMMU_IOAS_MAP_FIXED_IOVA | \
IOMMU_IOAS_MAP_WRITEABLE | \
IOMMU_IOAS_MAP_READABLE)); \
})
static int _test_ioctl_set_temp_memory_limit(int fd, unsigned int limit)
{
struct iommu_test_cmd memlimit_cmd = {
.size = sizeof(memlimit_cmd),
.op = IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
.memory_limit = { .limit = limit },
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT),
&memlimit_cmd);
}
#define test_ioctl_set_temp_memory_limit(limit) \
ASSERT_EQ(0, _test_ioctl_set_temp_memory_limit(self->fd, limit))
#define test_ioctl_set_default_memory_limit() \
test_ioctl_set_temp_memory_limit(65536)
static void teardown_iommufd(int fd, struct __test_metadata *_metadata)
{
struct iommu_test_cmd test_cmd = {
.size = sizeof(test_cmd),
.op = IOMMU_TEST_OP_MD_CHECK_REFS,
.check_refs = { .length = BUFFER_SIZE,
.uptr = (uintptr_t)buffer },
};
if (fd == -1)
return;
EXPECT_EQ(0, close(fd));
fd = open("/dev/iommu", O_RDWR);
EXPECT_NE(-1, fd);
EXPECT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS),
&test_cmd));
EXPECT_EQ(0, close(fd));
}
#define EXPECT_ERRNO(expected_errno, cmd) \
({ \
ASSERT_EQ(-1, cmd); \
EXPECT_EQ(expected_errno, errno); \
})
#endif
static int _test_cmd_get_hw_info(int fd, __u32 device_id, __u32 data_type,
void *data, size_t data_len,
uint32_t *capabilities, uint8_t *max_pasid)
{
struct iommu_test_hw_info *info = (struct iommu_test_hw_info *)data;
struct iommu_hw_info cmd = {
.size = sizeof(cmd),
.dev_id = device_id,
.data_len = data_len,
.in_data_type = data_type,
.data_uptr = (uint64_t)data,
.out_capabilities = 0,
};
int ret;
if (data_type != IOMMU_HW_INFO_TYPE_DEFAULT)
cmd.flags |= IOMMU_HW_INFO_FLAG_INPUT_TYPE;
ret = ioctl(fd, IOMMU_GET_HW_INFO, &cmd);
if (ret)
return ret;
assert(cmd.out_data_type == IOMMU_HW_INFO_TYPE_SELFTEST);
assert(cmd.data_len == sizeof(struct iommu_test_hw_info));
if (data_len > cmd.data_len) {
char *ptr = (char *)(data + cmd.data_len);
int idx = 0;
while (idx < data_len - cmd.data_len) {
assert(!*(ptr + idx));
idx++;
}
}
if (info) {
if (data_len >= offsetofend(struct iommu_test_hw_info, test_reg))
assert(info->test_reg == IOMMU_HW_INFO_SELFTEST_REGVAL);
if (data_len >= offsetofend(struct iommu_test_hw_info, flags))
assert(!info->flags);
}
if (max_pasid)
*max_pasid = cmd.out_max_pasid_log2;
if (capabilities)
*capabilities = cmd.out_capabilities;
return 0;
}
#define test_cmd_get_hw_info(device_id, data_type, data, data_len) \
ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, data_type, \
data, data_len, NULL, NULL))
#define test_err_get_hw_info(_errno, device_id, data_type, data, data_len) \
EXPECT_ERRNO(_errno, \
_test_cmd_get_hw_info(self->fd, device_id, data_type, \
data, data_len, NULL, NULL))
#define test_cmd_get_hw_capabilities(device_id, caps) \
ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, \
IOMMU_HW_INFO_TYPE_DEFAULT, NULL, \
0, &caps, NULL))
#define test_cmd_get_hw_info_pasid(device_id, max_pasid) \
ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, \
IOMMU_HW_INFO_TYPE_DEFAULT, NULL, \
0, NULL, max_pasid))
static int _test_ioctl_fault_alloc(int fd, __u32 *fault_id, __u32 *fault_fd)
{
struct iommu_fault_alloc cmd = {
.size = sizeof(cmd),
};
int ret;
ret = ioctl(fd, IOMMU_FAULT_QUEUE_ALLOC, &cmd);
if (ret)
return ret;
*fault_id = cmd.out_fault_id;
*fault_fd = cmd.out_fault_fd;
return 0;
}
#define test_ioctl_fault_alloc(fault_id, fault_fd) \
({ \
ASSERT_EQ(0, _test_ioctl_fault_alloc(self->fd, fault_id, \
fault_fd)); \
ASSERT_NE(0, *(fault_id)); \
ASSERT_NE(0, *(fault_fd)); \
})
static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 pasid,
__u32 fault_fd)
{
struct iommu_test_cmd trigger_iopf_cmd = {
.size = sizeof(trigger_iopf_cmd),
.op = IOMMU_TEST_OP_TRIGGER_IOPF,
.trigger_iopf = {
.dev_id = device_id,
.pasid = pasid,
.grpid = 0x2,
.perm = IOMMU_PGFAULT_PERM_READ | IOMMU_PGFAULT_PERM_WRITE,
.addr = 0xdeadbeaf,
},
};
struct iommu_hwpt_page_response response = {
.code = IOMMUFD_PAGE_RESP_SUCCESS,
};
struct iommu_hwpt_pgfault fault = {};
ssize_t bytes;
int ret;
ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_TRIGGER_IOPF), &trigger_iopf_cmd);
if (ret)
return ret;
bytes = read(fault_fd, &fault, sizeof(fault));
if (bytes <= 0)
return -EIO;
response.cookie = fault.cookie;
bytes = write(fault_fd, &response, sizeof(response));
if (bytes <= 0)
return -EIO;
return 0;
}
#define test_cmd_trigger_iopf(device_id, fault_fd) \
ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, 0x1, fault_fd))
#define test_cmd_trigger_iopf_pasid(device_id, pasid, fault_fd) \
ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, \
pasid, fault_fd))
static int _test_cmd_viommu_alloc(int fd, __u32 device_id, __u32 hwpt_id,
__u32 flags, __u32 type, void *data,
__u32 data_len, __u32 *viommu_id)
{
struct iommu_viommu_alloc cmd = {
.size = sizeof(cmd),
.flags = flags,
.type = type,
.dev_id = device_id,
.hwpt_id = hwpt_id,
.data_uptr = (uint64_t)data,
.data_len = data_len,
};
int ret;
ret = ioctl(fd, IOMMU_VIOMMU_ALLOC, &cmd);
if (ret)
return ret;
if (viommu_id)
*viommu_id = cmd.out_viommu_id;
return 0;
}
#define test_cmd_viommu_alloc(device_id, hwpt_id, type, data, data_len, \
viommu_id) \
ASSERT_EQ(0, _test_cmd_viommu_alloc(self->fd, device_id, hwpt_id, 0, \
type, data, data_len, viommu_id))
#define test_err_viommu_alloc(_errno, device_id, hwpt_id, type, data, \
data_len, viommu_id) \
EXPECT_ERRNO(_errno, \
_test_cmd_viommu_alloc(self->fd, device_id, hwpt_id, 0, \
type, data, data_len, viommu_id))
static int _test_cmd_vdevice_alloc(int fd, __u32 viommu_id, __u32 idev_id,
__u64 virt_id, __u32 *vdev_id)
{
struct iommu_vdevice_alloc cmd = {
.size = sizeof(cmd),
.dev_id = idev_id,
.viommu_id = viommu_id,
.virt_id = virt_id,
};
int ret;
ret = ioctl(fd, IOMMU_VDEVICE_ALLOC, &cmd);
if (ret)
return ret;
if (vdev_id)
*vdev_id = cmd.out_vdevice_id;
return 0;
}
#define test_cmd_vdevice_alloc(viommu_id, idev_id, virt_id, vdev_id) \
ASSERT_EQ(0, _test_cmd_vdevice_alloc(self->fd, viommu_id, idev_id, \
virt_id, vdev_id))
#define test_err_vdevice_alloc(_errno, viommu_id, idev_id, virt_id, vdev_id) \
EXPECT_ERRNO(_errno, \
_test_cmd_vdevice_alloc(self->fd, viommu_id, idev_id, \
virt_id, vdev_id))
static int _test_cmd_hw_queue_alloc(int fd, __u32 viommu_id, __u32 type,
__u32 idx, __u64 base_addr, __u64 length,
__u32 *hw_queue_id)
{
struct iommu_hw_queue_alloc cmd = {
.size = sizeof(cmd),
.viommu_id = viommu_id,
.type = type,
.index = idx,
.nesting_parent_iova = base_addr,
.length = length,
};
int ret;
ret = ioctl(fd, IOMMU_HW_QUEUE_ALLOC, &cmd);
if (ret)
return ret;
if (hw_queue_id)
*hw_queue_id = cmd.out_hw_queue_id;
return 0;
}
#define test_cmd_hw_queue_alloc(viommu_id, type, idx, base_addr, len, out_qid) \
ASSERT_EQ(0, _test_cmd_hw_queue_alloc(self->fd, viommu_id, type, idx, \
base_addr, len, out_qid))
#define test_err_hw_queue_alloc(_errno, viommu_id, type, idx, base_addr, len, \
out_qid) \
EXPECT_ERRNO(_errno, \
_test_cmd_hw_queue_alloc(self->fd, viommu_id, type, idx, \
base_addr, len, out_qid))
static int _test_cmd_veventq_alloc(int fd, __u32 viommu_id, __u32 type,
__u32 *veventq_id, __u32 *veventq_fd)
{
struct iommu_veventq_alloc cmd = {
.size = sizeof(cmd),
.type = type,
.veventq_depth = 2,
.viommu_id = viommu_id,
};
int ret;
ret = ioctl(fd, IOMMU_VEVENTQ_ALLOC, &cmd);
if (ret)
return ret;
if (veventq_id)
*veventq_id = cmd.out_veventq_id;
if (veventq_fd)
*veventq_fd = cmd.out_veventq_fd;
return 0;
}
#define test_cmd_veventq_alloc(viommu_id, type, veventq_id, veventq_fd) \
ASSERT_EQ(0, _test_cmd_veventq_alloc(self->fd, viommu_id, type, \
veventq_id, veventq_fd))
#define test_err_veventq_alloc(_errno, viommu_id, type, veventq_id, \
veventq_fd) \
EXPECT_ERRNO(_errno, \
_test_cmd_veventq_alloc(self->fd, viommu_id, type, \
veventq_id, veventq_fd))
static int _test_cmd_trigger_vevents(int fd, __u32 dev_id, __u32 nvevents)
{
struct iommu_test_cmd trigger_vevent_cmd = {
.size = sizeof(trigger_vevent_cmd),
.op = IOMMU_TEST_OP_TRIGGER_VEVENT,
.trigger_vevent = {
.dev_id = dev_id,
},
};
while (nvevents--) {
if (ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_TRIGGER_VEVENT),
&trigger_vevent_cmd))
return -1;
}
return 0;
}
#define test_cmd_trigger_vevents(dev_id, nvevents) \
ASSERT_EQ(0, _test_cmd_trigger_vevents(self->fd, dev_id, nvevents))
static int _test_cmd_read_vevents(int fd, __u32 event_fd, __u32 nvevents,
__u32 virt_id, int *prev_seq)
{
struct pollfd pollfd = { .fd = event_fd, .events = POLLIN };
struct iommu_viommu_event_selftest *event;
struct iommufd_vevent_header *hdr;
ssize_t bytes;
void *data;
int ret, i;
ret = poll(&pollfd, 1, 1000);
if (ret < 0)
return -1;
data = calloc(nvevents, sizeof(*hdr) + sizeof(*event));
if (!data) {
errno = ENOMEM;
return -1;
}
bytes = read(event_fd, data,
nvevents * (sizeof(*hdr) + sizeof(*event)));
if (bytes <= 0) {
errno = EFAULT;
ret = -1;
goto out_free;
}
for (i = 0; i < nvevents; i++) {
hdr = data + i * (sizeof(*hdr) + sizeof(*event));
if (hdr->flags & IOMMU_VEVENTQ_FLAG_LOST_EVENTS ||
hdr->sequence - *prev_seq > 1) {
*prev_seq = hdr->sequence;
errno = EOVERFLOW;
ret = -1;
goto out_free;
}
*prev_seq = hdr->sequence;
event = data + sizeof(*hdr);
if (event->virt_id != virt_id) {
errno = EINVAL;
ret = -1;
goto out_free;
}
}
ret = 0;
out_free:
free(data);
return ret;
}
#define test_cmd_read_vevents(event_fd, nvevents, virt_id, prev_seq) \
ASSERT_EQ(0, _test_cmd_read_vevents(self->fd, event_fd, nvevents, \
virt_id, prev_seq))
#define test_err_read_vevents(_errno, event_fd, nvevents, virt_id, prev_seq) \
EXPECT_ERRNO(_errno, \
_test_cmd_read_vevents(self->fd, event_fd, nvevents, \
virt_id, prev_seq))
static int _test_cmd_pasid_attach(int fd, __u32 stdev_id, __u32 pasid,
__u32 pt_id)
{
struct iommu_test_cmd test_attach = {
.size = sizeof(test_attach),
.op = IOMMU_TEST_OP_PASID_ATTACH,
.id = stdev_id,
.pasid_attach = {
.pasid = pasid,
.pt_id = pt_id,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_ATTACH),
&test_attach);
}
#define test_cmd_pasid_attach(pasid, hwpt_id) \
ASSERT_EQ(0, _test_cmd_pasid_attach(self->fd, self->stdev_id, \
pasid, hwpt_id))
#define test_err_pasid_attach(_errno, pasid, hwpt_id) \
EXPECT_ERRNO(_errno, \
_test_cmd_pasid_attach(self->fd, self->stdev_id, \
pasid, hwpt_id))
static int _test_cmd_pasid_replace(int fd, __u32 stdev_id, __u32 pasid,
__u32 pt_id)
{
struct iommu_test_cmd test_replace = {
.size = sizeof(test_replace),
.op = IOMMU_TEST_OP_PASID_REPLACE,
.id = stdev_id,
.pasid_replace = {
.pasid = pasid,
.pt_id = pt_id,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_REPLACE),
&test_replace);
}
#define test_cmd_pasid_replace(pasid, hwpt_id) \
ASSERT_EQ(0, _test_cmd_pasid_replace(self->fd, self->stdev_id, \
pasid, hwpt_id))
#define test_err_pasid_replace(_errno, pasid, hwpt_id) \
EXPECT_ERRNO(_errno, \
_test_cmd_pasid_replace(self->fd, self->stdev_id, \
pasid, hwpt_id))
static int _test_cmd_pasid_detach(int fd, __u32 stdev_id, __u32 pasid)
{
struct iommu_test_cmd test_detach = {
.size = sizeof(test_detach),
.op = IOMMU_TEST_OP_PASID_DETACH,
.id = stdev_id,
.pasid_detach = {
.pasid = pasid,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_DETACH),
&test_detach);
}
#define test_cmd_pasid_detach(pasid) \
ASSERT_EQ(0, _test_cmd_pasid_detach(self->fd, self->stdev_id, pasid))
static int test_cmd_pasid_check_hwpt(int fd, __u32 stdev_id, __u32 pasid,
__u32 hwpt_id)
{
struct iommu_test_cmd test_pasid_check = {
.size = sizeof(test_pasid_check),
.op = IOMMU_TEST_OP_PASID_CHECK_HWPT,
.id = stdev_id,
.pasid_check = {
.pasid = pasid,
.hwpt_id = hwpt_id,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_CHECK_HWPT),
&test_pasid_check);
}
