dev_dax
return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
if (dev_dax->nr_range == 0)
if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
last = &dev_dax->ranges[dev_dax->nr_range - 1];
for (i = 0; i < dev_dax->nr_range - 1; i++) {
struct dev_dax_range *dax_range = &dev_dax->ranges[i];
struct dev_dax *dev_dax, resource_size_t size)
resource_size_t dev_size = dev_dax_size(dev_dax);
struct device *dev = &dev_dax->dev;
return dev_dax_shrink(dev_dax, size);
if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
if (adjust_ok(dev_dax, res)) {
rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_region *dax_region = dev_dax->region;
if (!alloc_is_aligned(dev_dax, val)) {
rc = dev_dax_resize(dax_region, dev_dax, val);
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_region *dax_region = dev_dax->region;
if (alloc_is_aligned(dev_dax, to_alloc))
rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
struct dev_dax *dev_dax = to_dev_dax(dev);
return sysfs_emit(buf, "%d\n", dev_dax->align);
static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
struct device *dev = &dev_dax->dev;
for (i = 0; i < dev_dax->nr_range; i++) {
size_t len = range_len(&dev_dax->ranges[i].range);
if (!alloc_is_aligned(dev_dax, len)) {
__func__, dev_dax->align, i);
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_region *dax_region = dev_dax->region;
align_save = dev_dax->align;
dev_dax->align = val;
rc = dev_dax_validate_align(dev_dax);
dev_dax->align = align_save;
static int dev_dax_target_node(struct dev_dax *dev_dax)
struct dax_region *dax_region = dev_dax->region;
struct dev_dax *dev_dax = to_dev_dax(dev);
return sysfs_emit(buf, "%d\n", dev_dax_target_node(dev_dax));
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_region *dax_region = dev_dax->region;
if (dev_dax->nr_range < 1)
start = dev_dax->ranges[0].range.start;
struct dev_dax *dev_dax = to_dev_dax(dev);
return sysfs_emit(buf, "%d\n", dev_dax->memmap_on_memory);
struct dev_dax *dev_dax = to_dev_dax(dev);
if (dev_dax->memmap_on_memory != val && dev->driver &&
dev_dax->memmap_on_memory = val;
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_region *dax_region = dev_dax->region;
if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_device *dax_dev = dev_dax->dax_dev;
free_dev_dax_id(dev_dax);
kfree(dev_dax->pgmap);
kfree(dev_dax);
__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
for (int i = 0; i < dev_dax->nr_range; i++) {
struct dev_dax_range *dax_range = &dev_dax->ranges[i];
static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data)
struct dev_dax *dev_dax;
dev_dax = kzalloc_obj(*dev_dax);
if (!dev_dax)
dev_dax->region = dax_region;
dev_dax->id = data->id;
rc = alloc_dev_dax_id(dev_dax);
dev = &dev_dax->dev;
dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
dev_dax->pgmap = kmemdup(data->pgmap,
if (!dev_dax->pgmap) {
dax_dev = alloc_dax(dev_dax, NULL);
dev_dax->dax_dev = dax_dev;
dev_dax->target_node = dax_region->target_node;
dev_dax->align = dax_region->align;
ida_init(&dev_dax->ida);
dev_dax->memmap_on_memory = data->memmap_on_memory;
kill_dev_dax(dev_dax);
if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
rc = devm_register_dax_mapping(dev_dax, 0);
return dev_dax;
kfree(dev_dax->pgmap);
free_dev_dax_ranges(dev_dax);
free_dev_dax_id(dev_dax);
kfree(dev_dax);
struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
struct dev_dax *dev_dax;
dev_dax = __devm_create_dev_dax(data);
return dev_dax;
bool static_dev_dax(struct dev_dax *dev_dax)
return is_static(dev_dax->region);
static u64 dev_dax_size(struct dev_dax *dev_dax)
for (i = 0; i < dev_dax->nr_range; i++)
size += range_len(&dev_dax->ranges[i].range);
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_region *dax_region = dev_dax->region;
size = dev_dax_size(dev_dax);
if (size == 0 || dev_dax->id < 0)
rc = dax_drv->probe(dev_dax);
struct dev_dax *dev_dax = to_dev_dax(dev);
dax_drv->remove(dev_dax);
static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data);
struct dev_dax *dev_dax = __devm_create_dev_dax(&data);
if (IS_ERR(dev_dax))
rc = PTR_ERR(dev_dax);
dax_region->seed = &dev_dax->dev;
dax_region->youngest = &dev_dax->dev;
void kill_dev_dax(struct dev_dax *dev_dax)
struct dax_device *dax_dev = dev_dax->dax_dev;
if (!static_dev_dax(dev_dax))
dev_dax->pgmap = NULL;
static void trim_dev_dax_range(struct dev_dax *dev_dax)
int i = dev_dax->nr_range - 1;
struct range *range = &dev_dax->ranges[i].range;
struct dax_region *dax_region = dev_dax->region;
dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i,
if (--dev_dax->nr_range == 0) {
kfree(dev_dax->ranges);
dev_dax->ranges = NULL;
static void free_dev_dax_ranges(struct dev_dax *dev_dax)
while (dev_dax->nr_range)
trim_dev_dax_range(dev_dax);
struct dev_dax *dev_dax = to_dev_dax(dev);
kill_dev_dax(dev_dax);
free_dev_dax_ranges(dev_dax);
static int __free_dev_dax_id(struct dev_dax *dev_dax)
int rc = dev_dax->id;
if (!dev_dax->dyn_id || dev_dax->id < 0)
dax_region = dev_dax->region;
ida_free(&dax_region->ida, dev_dax->id);
dev_dax->id = -1;
static int free_dev_dax_id(struct dev_dax *dev_dax)
rc = __free_dev_dax_id(dev_dax);
static int alloc_dev_dax_id(struct dev_dax *dev_dax)
struct dax_region *dax_region = dev_dax->region;
dev_dax->dyn_id = true;
dev_dax->id = id;
struct dev_dax *dev_dax;
dev_dax = to_dev_dax(victim);
if (victim->driver || dev_dax_size(dev_dax))
if (dev_dax->id > 0) {
do_del = __free_dev_dax_id(dev_dax) >= 0;
struct dev_dax *dev_dax = to_dev_dax(parent);
ida_free(&dev_dax->ida, mapping->id);
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dev_dax *dev_dax = to_dev_dax(dev->parent);
dev_dax->ranges[mapping->range_id].mapping = NULL;
if (dev_dax->region->res.flags & IORESOURCE_DAX_KMEM)
struct dev_dax *dev_dax = to_dev_dax(dev->parent);
return &dev_dax->ranges[mapping->range_id];
static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
struct dax_region *dax_region = dev_dax->region;
if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
dev_dax->ranges[range_id].mapping = mapping;
dev->parent = &dev_dax->dev;
static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
struct dax_region *dax_region = dev_dax->region;
struct device *dev = &dev_dax->dev;
if (dev_WARN_ONCE(dev, dev_dax->nr_range,
ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
* (dev_dax->nr_range + 1), GFP_KERNEL);
for (i = 0; i < dev_dax->nr_range; i++)
dev_dax->ranges = ranges;
ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
if (!device_is_registered(&dev_dax->dev))
rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
trim_dev_dax_range(dev_dax);
static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
int last_range = dev_dax->nr_range - 1;
struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
struct device *dev = &dev_dax->dev;
struct dev_dax *dev_dax = to_dev_dax(dev);
size = dev_dax_size(dev_dax);
static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
struct dax_region *dax_region = dev_dax->region;
struct device *dev = &dev_dax->dev;
for (i = dev_dax->nr_range - 1; i >= 0; i--) {
struct range *range = &dev_dax->ranges[i].range;
struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
trim_dev_dax_range(dev_dax);
if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
struct dev_dax;
struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data);
int (*probe)(struct dev_dax *dev);
void (*remove)(struct dev_dax *dev);
void kill_dev_dax(struct dev_dax *dev_dax);
bool static_dev_dax(struct dev_dax *dev_dax);
static inline struct dev_dax *to_dev_dax(struct device *dev)
return container_of(dev, struct dev_dax, dev);
phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, unsigned long size);
if (fault_size != dev_dax->align)
phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
struct device *dev = &dev_dax->dev;
if (check_vma(dev_dax, vmf->vma, __func__))
if (dev_dax->align > PMD_SIZE) {
dev_dax->align, fault_size);
if (fault_size < dev_dax->align)
else if (fault_size > dev_dax->align)
phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
static int __check_vma(struct dev_dax *dev_dax, vma_flags_t flags,
static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
struct device *dev = &dev_dax->dev;
if (check_vma(dev_dax, vmf->vma, __func__))
if (dev_dax->align > PUD_SIZE) {
dev_dax->align, fault_size);
if (fault_size < dev_dax->align)
else if (fault_size > dev_dax->align)
phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
struct device *dev = &dev_dax->dev;
static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
struct dev_dax *dev_dax = filp->private_data;
dev_dbg(&dev_dax->dev, "%s: op=%s addr=%#lx order=%d\n", current->comm,
if (!dax_alive(dev_dax->dax_dev))
rc = __dev_dax_pte_fault(dev_dax, vmf);
rc = __dev_dax_pmd_fault(dev_dax, vmf);
rc = __dev_dax_pud_fault(dev_dax, vmf);
struct dev_dax *dev_dax = filp->private_data;
if (!IS_ALIGNED(addr, dev_dax->align))
struct dev_dax *dev_dax = filp->private_data;
return dev_dax->align;
struct dev_dax *dev_dax = filp->private_data;
dev_dbg(&dev_dax->dev, "trace\n");
rc = __check_vma(dev_dax, desc->vma_flags, desc->start, desc->end, filp,
struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
if (!dev_dax || addr)
align = dev_dax->align;
struct dev_dax *dev_dax = dax_get_private(dax_dev);
dev_dbg(&dev_dax->dev, "trace\n");
mask = dev_dax->align - 1;
filp->private_data = dev_dax;
struct dev_dax *dev_dax = filp->private_data;
dev_dbg(&dev_dax->dev, "trace\n");
static void dev_dax_kill(void *dev_dax)
kill_dev_dax(dev_dax);
static int dev_dax_probe(struct dev_dax *dev_dax)
struct dax_device *dax_dev = dev_dax->dax_dev;
struct device *dev = &dev_dax->dev;
if (static_dev_dax(dev_dax)) {
if (dev_dax->nr_range > 1) {
pgmap = dev_dax->pgmap;
if (dev_dax->pgmap) {
struct_size(pgmap, ranges, dev_dax->nr_range - 1),
pgmap->nr_range = dev_dax->nr_range;
dev_dax->pgmap = pgmap;
for (i = 0; i < dev_dax->nr_range; i++) {
struct range *range = &dev_dax->ranges[i].range;
for (i = 0; i < dev_dax->nr_range; i++) {
struct range *range = &dev_dax->ranges[i].range;
if (dev_dax->align > PAGE_SIZE)
order_base_2(dev_dax->align >> PAGE_SHIFT);
return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
return __check_vma(dev_dax, vma->flags, vma->vm_start, vma->vm_end,
struct dev_dax *dev_dax = filp->private_data;
if (dev_dax->pgmap->vmemmap_shift)
static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
struct device *dev = &dev_dax->dev;
if (check_vma(dev_dax, vmf->vma, __func__))
if (dev_dax->align > PAGE_SIZE) {
dev_dax->align, fault_size);
static void fsdev_kill(void *dev_dax)
kill_dev_dax(dev_dax);
struct dev_dax *dev_dax = data;
dax_set_ops(dev_dax->dax_dev, NULL);
struct dev_dax *dev_dax = pgmap->owner;
u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start;
return dax_holder_notify_failure(dev_dax->dax_dev, offset,
static void fsdev_clear_folio_state(struct dev_dax *dev_dax)
for (int i = 0; i < dev_dax->nr_range; i++) {
struct range *range = &dev_dax->ranges[i].range;
struct dev_dax *dev_dax = dax_get_private(dax_dev);
filp->private_data = dev_dax;
static int fsdev_dax_probe(struct dev_dax *dev_dax)
struct dax_device *dax_dev = dev_dax->dax_dev;
struct device *dev = &dev_dax->dev;
if (static_dev_dax(dev_dax)) {
if (dev_dax->nr_range > 1) {
pgmap = dev_dax->pgmap;
if (dev_dax->pgmap) {
pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1);
pgmap->nr_range = dev_dax->nr_range;
dev_dax->pgmap = pgmap;
for (i = 0; i < dev_dax->nr_range; i++) {
struct range *range = &dev_dax->ranges[i].range;
for (i = 0; i < dev_dax->nr_range; i++) {
struct range *range = &dev_dax->ranges[i].range;
dev_dax->cached_size = 0;
for (i = 0; i < dev_dax->nr_range; i++)
dev_dax->cached_size += range_len(&dev_dax->ranges[i].range);
pgmap->owner = dev_dax;
fsdev_clear_folio_state(dev_dax);
dev_dax);
if (pgmap->range.start != dev_dax->ranges[0].range.start) {
u64 phys = dev_dax->ranges[0].range.start;
u64 pgmap_phys = dev_dax->pgmap[0].range.start;
dev_dax->virt_addr = addr + data_offset;
rc = devm_add_action_or_reset(dev, fsdev_clear_ops, dev_dax);
return devm_add_action_or_reset(dev, fsdev_kill, dev_dax);
struct dev_dax *dev_dax = dax_get_private(dax_dev);
void *virt_addr = dev_dax->virt_addr + offset;
phys = dax_pgoff_to_phys(dev_dax, pgoff, size);
dev_dbg(&dev_dax->dev,
return PHYS_PFN(min(size, dev_dax->cached_size - offset));
orig_len += range_len(&dev_dax->ranges[i].range);
rc = dax_kmem_range(dev_dax, i, &range);
data = kzalloc_flex(*data, res, dev_dax->nr_range);
for (i = 0; i < dev_dax->nr_range; i++) {
rc = dax_kmem_range(dev_dax, i, &range);
if (dev_dax->memmap_on_memory)
static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
int node = dev_dax->target_node;
struct device *dev = &dev_dax->dev;
for (i = 0; i < dev_dax->nr_range; i++) {
rc = dax_kmem_range(dev_dax, i, &range);
if (success >= dev_dax->nr_range) {
static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r)
struct dev_dax_range *dax_range = &dev_dax->ranges[i];
static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
struct device *dev = &dev_dax->dev;
numa_node = dev_dax->target_node;
for (i = 0; i < dev_dax->nr_range; i++) {
static struct dev_dax *__dax_pmem_probe(struct device *dev)
struct dev_dax *dev_dax;
dev_dax = dax_get_private(dax_dev);
if (!dev_dax) {
device_lock(&dev_dax->dev);
if (!dev_dax->dev.driver) {
device_unlock(&dev_dax->dev);
dax_drv = to_dax_drv(dev_dax->dev.driver);
device_unlock(&dev_dax->dev);
device_unlock(&dev_dax->dev);
for (i = 0; i < dev_dax->nr_range; i++) {
struct dev_dax_range *dax_range = &dev_dax->ranges[i];
if (dev_dax->region->align > PAGE_SIZE)
phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,