diff options
Diffstat (limited to 'drivers/dax')
| -rw-r--r-- | drivers/dax/Kconfig | 90 | ||||
| -rw-r--r-- | drivers/dax/Makefile | 15 | ||||
| -rw-r--r-- | drivers/dax/bus.c | 1625 | ||||
| -rw-r--r-- | drivers/dax/bus.h | 79 | ||||
| -rw-r--r-- | drivers/dax/cxl.c | 75 | ||||
| -rw-r--r-- | drivers/dax/dax-private.h | 138 | ||||
| -rw-r--r-- | drivers/dax/device.c | 476 | ||||
| -rw-r--r-- | drivers/dax/fsdev.c | 349 | ||||
| -rw-r--r-- | drivers/dax/hmem/Makefile | 7 | ||||
| -rw-r--r-- | drivers/dax/hmem/device.c | 103 | ||||
| -rw-r--r-- | drivers/dax/hmem/hmem.c | 257 | ||||
| -rw-r--r-- | drivers/dax/kmem.c | 319 | ||||
| -rw-r--r-- | drivers/dax/pmem.c | 99 | ||||
| -rw-r--r-- | drivers/dax/super.c | 712 |
14 files changed, 4344 insertions, 0 deletions
diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig new file mode 100644 index 000000000..602f9a083 --- /dev/null +++ b/drivers/dax/Kconfig @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig DAX + tristate "DAX: direct access to differentiated memory" + default m if NVDIMM_DAX + +if DAX + +config DEV_DAX + tristate "Device DAX: direct access mapping device" + depends on TRANSPARENT_HUGEPAGE + help + Support raw access to differentiated (persistence, bandwidth, + latency...) memory via an mmap(2) capable character + device. Platform firmware or a device driver may identify a + platform memory resource that is differentiated from the + baseline memory pool. Mappings of a /dev/daxX.Y device impose + restrictions that make the mapping behavior deterministic. + +config DEV_DAX_PMEM + tristate "PMEM DAX: direct access to persistent memory" + depends on LIBNVDIMM && NVDIMM_DAX && DEV_DAX + default DEV_DAX + help + Support raw access to persistent memory. Note that this + driver consumes memory ranges allocated and exported by the + libnvdimm sub-system. + + Say M if unsure + +config DEV_DAX_HMEM + tristate "HMEM DAX: direct access to 'specific purpose' memory" + depends on EFI_SOFT_RESERVE + select NUMA_KEEP_MEMINFO if NUMA_MEMBLKS + default DEV_DAX + depends on CXL_ACPI || !CXL_ACPI + depends on CXL_PCI || !CXL_PCI + depends on CXL_BUS || !CXL_BUS + help + EFI 2.8 platforms, and others, may advertise 'specific purpose' + memory. For example, a high bandwidth memory pool. The + indication from platform firmware is meant to reserve the + memory from typical usage by default. This driver creates + device-dax instances for these memory ranges, and that also + enables the possibility to assign them to the DEV_DAX_KMEM + driver to override the reservation and add them to kernel + "System RAM" pool. + + Say M if unsure. + +config DEV_DAX_CXL + tristate "CXL DAX: direct access to CXL RAM regions" + depends on CXL_BUS && CXL_REGION && DEV_DAX + default CXL_REGION && DEV_DAX + depends on DEV_DAX_HMEM || !DEV_DAX_HMEM + help + CXL RAM regions are either mapped by platform-firmware + and published in the initial system-memory map as "System RAM", mapped + by platform-firmware as "Soft Reserved", or dynamically provisioned + after boot by the CXL driver. In the latter two cases a device-dax + instance is created to access that unmapped-by-default address range. + Per usual it can remain as dedicated access via a device interface, or + converted to "System RAM" via the dax_kmem facility. + +config DEV_DAX_HMEM_DEVICES + depends on DEV_DAX_HMEM && DAX + def_bool y + +config DEV_DAX_FSDEV + tristate + depends on DEV_DAX && FS_DAX + default DEV_DAX + +config DEV_DAX_KMEM + tristate "KMEM DAX: map dax-devices as System-RAM" + default DEV_DAX + depends on DEV_DAX + depends on MEMORY_HOTPLUG # for add_memory() and friends + help + Support access to persistent, or other performance + differentiated memory as if it were System RAM. This allows + easier use of persistent memory by unmodified applications, or + adds core kernel memory services to heterogeneous memory types + (HMEM) marked "reserved" by platform firmware. + + To use this feature, a DAX device must be unbound from the + device_dax driver and bound to this kmem driver on each boot. + + Say N if unsure. + +endif diff --git a/drivers/dax/Makefile b/drivers/dax/Makefile new file mode 100644 index 000000000..c0bc6eb1e --- /dev/null +++ b/drivers/dax/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += hmem/ +obj-$(CONFIG_DAX) += dax.o +obj-$(CONFIG_DEV_DAX) += device_dax.o +obj-$(CONFIG_DEV_DAX_KMEM) += kmem.o +obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o +obj-$(CONFIG_DEV_DAX_CXL) += dax_cxl.o +obj-$(CONFIG_DEV_DAX_FSDEV) += fsdev_dax.o + +dax-y := super.o +dax-y += bus.o +device_dax-y := device.o +dax_pmem-y := pmem.o +dax_cxl-y := cxl.o +fsdev_dax-y := fsdev.o diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c new file mode 100644 index 000000000..b809e1a26 --- /dev/null +++ b/drivers/dax/bus.c @@ -0,0 +1,1625 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> +#include <linux/device.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/dax.h> +#include <linux/io.h> +#include "dax-private.h" +#include "bus.h" + +static struct resource dax_regions = DEFINE_RES_MEM_NAMED(0, -1, "DAX Regions"); +static DEFINE_MUTEX(dax_bus_lock); + +/* + * All changes to the dax region configuration occur with this lock held + * for write. + */ +DECLARE_RWSEM(dax_region_rwsem); + +/* + * All changes to the dax device configuration occur with this lock held + * for write. + */ +DECLARE_RWSEM(dax_dev_rwsem); + +#define DAX_NAME_LEN 30 +struct dax_id { + struct list_head list; + char dev_name[DAX_NAME_LEN]; +}; + +static int dax_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + /* + * We only ever expect to handle device-dax instances, i.e. the + * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero + */ + return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0); +} + +static struct dax_id *__dax_match_id(const struct dax_device_driver *dax_drv, + const char *dev_name) +{ + struct dax_id *dax_id; + + lockdep_assert_held(&dax_bus_lock); + + list_for_each_entry(dax_id, &dax_drv->ids, list) + if (sysfs_streq(dax_id->dev_name, dev_name)) + return dax_id; + return NULL; +} + +static int dax_match_id(const struct dax_device_driver *dax_drv, struct device *dev) +{ + int match; + + mutex_lock(&dax_bus_lock); + match = !!__dax_match_id(dax_drv, dev_name(dev)); + mutex_unlock(&dax_bus_lock); + + return match; +} + +static int dax_match_type(const struct dax_device_driver *dax_drv, struct device *dev) +{ + enum dax_driver_type type = DAXDRV_DEVICE_TYPE; + struct dev_dax *dev_dax = to_dev_dax(dev); + + if (dev_dax->region->res.flags & IORESOURCE_DAX_KMEM) + type = DAXDRV_KMEM_TYPE; + + if (dax_drv->type == type) + return 1; + + /* default to device mode if dax_kmem is disabled */ + if (dax_drv->type == DAXDRV_DEVICE_TYPE && + !IS_ENABLED(CONFIG_DEV_DAX_KMEM)) + return 1; + + return 0; +} + +enum id_action { + ID_REMOVE, + ID_ADD, +}; + +static ssize_t do_id_store(struct device_driver *drv, const char *buf, + size_t count, enum id_action action) +{ + struct dax_device_driver *dax_drv = to_dax_drv(drv); + unsigned int region_id, id; + char devname[DAX_NAME_LEN]; + struct dax_id *dax_id; + ssize_t rc = count; + int fields; + + fields = sscanf(buf, "dax%d.%d", ®ion_id, &id); + if (fields != 2) + return -EINVAL; + sprintf(devname, "dax%d.%d", region_id, id); + if (!sysfs_streq(buf, devname)) + return -EINVAL; + + mutex_lock(&dax_bus_lock); + dax_id = __dax_match_id(dax_drv, buf); + if (!dax_id) { + if (action == ID_ADD) { + dax_id = kzalloc_obj(*dax_id); + if (dax_id) { + strscpy(dax_id->dev_name, buf, DAX_NAME_LEN); + list_add(&dax_id->list, &dax_drv->ids); + } else + rc = -ENOMEM; + } + } else if (action == ID_REMOVE) { + list_del(&dax_id->list); + kfree(dax_id); + } + mutex_unlock(&dax_bus_lock); + + if (rc < 0) + return rc; + if (action == ID_ADD) + rc = driver_attach(drv); + if (rc) + return rc; + return count; +} + +static ssize_t new_id_store(struct device_driver *drv, const char *buf, + size_t count) +{ + return do_id_store(drv, buf, count, ID_ADD); +} +static DRIVER_ATTR_WO(new_id); + +static ssize_t remove_id_store(struct device_driver *drv, const char *buf, + size_t count) +{ + return do_id_store(drv, buf, count, ID_REMOVE); +} +static DRIVER_ATTR_WO(remove_id); + +static struct attribute *dax_drv_attrs[] = { + &driver_attr_new_id.attr, + &driver_attr_remove_id.attr, + NULL, +}; +ATTRIBUTE_GROUPS(dax_drv); + +static int dax_bus_match(struct device *dev, const struct device_driver *drv); + +/* + * Static dax regions are regions created by an external subsystem + * nvdimm where a single range is assigned. Its boundaries are by the external + * subsystem and are usually limited to one physical memory range. For example, + * for PMEM it is usually defined by NVDIMM Namespace boundaries (i.e. a + * single contiguous range) + * + * On dynamic dax regions, the assigned region can be partitioned by dax core + * into multiple subdivisions. A subdivision is represented into one + * /dev/daxN.M device composed by one or more potentially discontiguous ranges. + * + * When allocating a dax region, drivers must set whether it's static + * (IORESOURCE_DAX_STATIC). On static dax devices, the @pgmap is pre-assigned + * to dax core when calling devm_create_dev_dax(), whereas in dynamic dax + * devices it is NULL but afterwards allocated by dax core on device ->probe(). + * Care is needed to make sure that dynamic dax devices are torn down with a + * cleared @pgmap field (see kill_dev_dax()). + */ +static bool is_static(struct dax_region *dax_region) +{ + return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0; +} + +bool static_dev_dax(struct dev_dax *dev_dax) +{ + return is_static(dev_dax->region); +} +EXPORT_SYMBOL_GPL(static_dev_dax); + +static u64 dev_dax_size(struct dev_dax *dev_dax) +{ + u64 size = 0; + int i; + + lockdep_assert_held(&dax_dev_rwsem); + + for (i = 0; i < dev_dax->nr_range; i++) + size += range_len(&dev_dax->ranges[i].range); + + return size; +} + +static int dax_bus_probe(struct device *dev) +{ + struct dax_device_driver *dax_drv = to_dax_drv(dev->driver); + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_region *dax_region = dev_dax->region; + int rc; + u64 size; + + rc = down_read_interruptible(&dax_dev_rwsem); + if (rc) + return rc; + size = dev_dax_size(dev_dax); + up_read(&dax_dev_rwsem); + + if (size == 0 || dev_dax->id < 0) + return -ENXIO; + + rc = dax_drv->probe(dev_dax); + + if (rc || is_static(dax_region)) + return rc; + + /* + * Track new seed creation only after successful probe of the + * previous seed. + */ + if (dax_region->seed == dev) + dax_region->seed = NULL; + + return 0; +} + +static void dax_bus_remove(struct device *dev) +{ + struct dax_device_driver *dax_drv = to_dax_drv(dev->driver); + struct dev_dax *dev_dax = to_dev_dax(dev); + + if (dax_drv->remove) + dax_drv->remove(dev_dax); +} + +static const struct bus_type dax_bus_type = { + .name = "dax", + .uevent = dax_bus_uevent, + .match = dax_bus_match, + .probe = dax_bus_probe, + .remove = dax_bus_remove, + .drv_groups = dax_drv_groups, +}; + +static int dax_bus_match(struct device *dev, const struct device_driver *drv) +{ + const struct dax_device_driver *dax_drv = to_dax_drv(drv); + + if (dax_match_id(dax_drv, dev)) + return 1; + return dax_match_type(dax_drv, dev); +} + +/* + * Rely on the fact that drvdata is set before the attributes are + * registered, and that the attributes are unregistered before drvdata + * is cleared to assume that drvdata is always valid. + */ +static ssize_t id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", dax_region->id); +} +static DEVICE_ATTR_RO(id); + +static ssize_t region_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%llu\n", + (unsigned long long)resource_size(&dax_region->res)); +} +static struct device_attribute dev_attr_region_size = __ATTR(size, 0444, + region_size_show, NULL); + +static ssize_t region_align_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%u\n", dax_region->align); +} +static struct device_attribute dev_attr_region_align = + __ATTR(align, 0400, region_align_show, NULL); + +#define for_each_dax_region_resource(dax_region, res) \ + for (res = (dax_region)->res.child; res; res = res->sibling) + +static unsigned long long dax_region_avail_size(struct dax_region *dax_region) +{ + resource_size_t size = resource_size(&dax_region->res); + struct resource *res; + + lockdep_assert_held(&dax_region_rwsem); + + for_each_dax_region_resource(dax_region, res) + size -= resource_size(res); + return size; +} + +static ssize_t available_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + unsigned long long size; + int rc; + + rc = down_read_interruptible(&dax_region_rwsem); + if (rc) + return rc; + size = dax_region_avail_size(dax_region); + up_read(&dax_region_rwsem); + + return sysfs_emit(buf, "%llu\n", size); +} +static DEVICE_ATTR_RO(available_size); + +static ssize_t seed_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + struct device *seed; + ssize_t rc; + + if (is_static(dax_region)) + return -EINVAL; + + rc = down_read_interruptible(&dax_region_rwsem); + if (rc) + return rc; + seed = dax_region->seed; + rc = sysfs_emit(buf, "%s\n", seed ? dev_name(seed) : ""); + up_read(&dax_region_rwsem); + + return rc; +} +static DEVICE_ATTR_RO(seed); + +static ssize_t create_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + struct device *youngest; + ssize_t rc; + + if (is_static(dax_region)) + return -EINVAL; + + rc = down_read_interruptible(&dax_region_rwsem); + if (rc) + return rc; + youngest = dax_region->youngest; + rc = sysfs_emit(buf, "%s\n", youngest ? dev_name(youngest) : ""); + up_read(&dax_region_rwsem); + + return rc; +} + +static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data); + +static ssize_t create_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + unsigned long long avail; + ssize_t rc; + int val; + + if (is_static(dax_region)) + return -EINVAL; + + rc = kstrtoint(buf, 0, &val); + if (rc) + return rc; + if (val != 1) + return -EINVAL; + + rc = down_write_killable(&dax_region_rwsem); + if (rc) + return rc; + avail = dax_region_avail_size(dax_region); + if (avail == 0) + rc = -ENOSPC; + else { + struct dev_dax_data data = { + .dax_region = dax_region, + .size = 0, + .id = -1, + .memmap_on_memory = false, + }; + struct dev_dax *dev_dax = __devm_create_dev_dax(&data); + + if (IS_ERR(dev_dax)) + rc = PTR_ERR(dev_dax); + else { + /* + * In support of crafting multiple new devices + * simultaneously multiple seeds can be created, + * but only the first one that has not been + * successfully bound is tracked as the region + * seed. + */ + if (!dax_region->seed) + dax_region->seed = &dev_dax->dev; + dax_region->youngest = &dev_dax->dev; + rc = len; + } + } + up_write(&dax_region_rwsem); + + return rc; +} +static DEVICE_ATTR_RW(create); + +void kill_dev_dax(struct dev_dax *dev_dax) +{ + struct dax_device *dax_dev = dev_dax->dax_dev; + struct inode *inode = dax_inode(dax_dev); + + kill_dax(dax_dev); + unmap_mapping_range(inode->i_mapping, 0, 0, 1); + + /* + * Dynamic dax region have the pgmap allocated via dev_kzalloc() + * and thus freed by devm. Clear the pgmap to not have stale pgmap + * ranges on probe() from previous reconfigurations of region devices. + */ + if (!static_dev_dax(dev_dax)) + dev_dax->pgmap = NULL; +} +EXPORT_SYMBOL_GPL(kill_dev_dax); + +static void trim_dev_dax_range(struct dev_dax *dev_dax) +{ + int i = dev_dax->nr_range - 1; + struct range *range = &dev_dax->ranges[i].range; + struct dax_region *dax_region = dev_dax->region; + + lockdep_assert_held_write(&dax_region_rwsem); + dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i, + (unsigned long long)range->start, + (unsigned long long)range->end); + + __release_region(&dax_region->res, range->start, range_len(range)); + if (--dev_dax->nr_range == 0) { + kfree(dev_dax->ranges); + dev_dax->ranges = NULL; + } +} + +static void free_dev_dax_ranges(struct dev_dax *dev_dax) +{ + while (dev_dax->nr_range) + trim_dev_dax_range(dev_dax); +} + +static void unregister_dev_dax(void *dev) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + + dev_dbg(dev, "%s\n", __func__); + + down_write(&dax_region_rwsem); + kill_dev_dax(dev_dax); + device_del(dev); + free_dev_dax_ranges(dev_dax); + put_device(dev); + up_write(&dax_region_rwsem); +} + +static void dax_region_free(struct kref *kref) +{ + struct dax_region *dax_region; + + dax_region = container_of(kref, struct dax_region, kref); + kfree(dax_region); +} + +static void dax_region_put(struct dax_region *dax_region) +{ + kref_put(&dax_region->kref, dax_region_free); +} + +/* a return value >= 0 indicates this invocation invalidated the id */ +static int __free_dev_dax_id(struct dev_dax *dev_dax) +{ + struct dax_region *dax_region; + int rc = dev_dax->id; + + lockdep_assert_held_write(&dax_dev_rwsem); + + if (!dev_dax->dyn_id || dev_dax->id < 0) + return -1; + dax_region = dev_dax->region; + ida_free(&dax_region->ida, dev_dax->id); + dax_region_put(dax_region); + dev_dax->id = -1; + return rc; +} + +static int free_dev_dax_id(struct dev_dax *dev_dax) +{ + int rc; + + rc = down_write_killable(&dax_dev_rwsem); + if (rc) + return rc; + rc = __free_dev_dax_id(dev_dax); + up_write(&dax_dev_rwsem); + return rc; +} + +static int alloc_dev_dax_id(struct dev_dax *dev_dax) +{ + struct dax_region *dax_region = dev_dax->region; + int id; + + id = ida_alloc(&dax_region->ida, GFP_KERNEL); + if (id < 0) + return id; + kref_get(&dax_region->kref); + dev_dax->dyn_id = true; + dev_dax->id = id; + return id; +} + +static ssize_t delete_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + struct dev_dax *dev_dax; + struct device *victim; + bool do_del = false; + int rc; + + if (is_static(dax_region)) + return -EINVAL; + + victim = device_find_child_by_name(dax_region->dev, buf); + if (!victim) + return -ENXIO; + + device_lock(dev); + device_lock(victim); + dev_dax = to_dev_dax(victim); + down_write(&dax_dev_rwsem); + if (victim->driver || dev_dax_size(dev_dax)) + rc = -EBUSY; + else { + /* + * Invalidate the device so it does not become active + * again, but always preserve device-id-0 so that + * /sys/bus/dax/ is guaranteed to be populated while any + * dax_region is registered. + */ + if (dev_dax->id > 0) { + do_del = __free_dev_dax_id(dev_dax) >= 0; + rc = len; + if (dax_region->seed == victim) + dax_region->seed = NULL; + if (dax_region->youngest == victim) + dax_region->youngest = NULL; + } else + rc = -EBUSY; + } + up_write(&dax_dev_rwsem); + device_unlock(victim); + + /* won the race to invalidate the device, clean it up */ + if (do_del) + devm_release_action(dev, unregister_dev_dax, victim); + device_unlock(dev); + put_device(victim); + + return rc; +} +static DEVICE_ATTR_WO(delete); + +static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a, + int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct dax_region *dax_region = dev_get_drvdata(dev); + + if (is_static(dax_region)) + if (a == &dev_attr_available_size.attr + || a == &dev_attr_create.attr + || a == &dev_attr_seed.attr + || a == &dev_attr_delete.attr) + return 0; + return a->mode; +} + +static struct attribute *dax_region_attributes[] = { + &dev_attr_available_size.attr, + &dev_attr_region_size.attr, + &dev_attr_region_align.attr, + &dev_attr_create.attr, + &dev_attr_seed.attr, + &dev_attr_delete.attr, + &dev_attr_id.attr, + NULL, +}; + +static const struct attribute_group dax_region_attribute_group = { + .name = "dax_region", + .attrs = dax_region_attributes, + .is_visible = dax_region_visible, +}; + +static const struct attribute_group *dax_region_attribute_groups[] = { + &dax_region_attribute_group, + NULL, +}; + +static void dax_region_unregister(void *region) +{ + struct dax_region *dax_region = region; + + sysfs_remove_groups(&dax_region->dev->kobj, + dax_region_attribute_groups); + release_resource(&dax_region->res); + dax_region_put(dax_region); +} + +struct dax_region *alloc_dax_region(struct device *parent, int region_id, + struct range *range, int target_node, unsigned int align, + unsigned long flags) +{ + struct dax_region *dax_region; + int rc; + + /* + * The DAX core assumes that it can store its private data in + * parent->driver_data. This WARN is a reminder / safeguard for + * developers of device-dax drivers. + */ + if (dev_get_drvdata(parent)) { + dev_WARN(parent, "dax core failed to setup private data\n"); + return NULL; + } + + if (!IS_ALIGNED(range->start, align) + || !IS_ALIGNED(range_len(range), align)) + return NULL; + + dax_region = kzalloc_obj(*dax_region); + if (!dax_region) + return NULL; + + dev_set_drvdata(parent, dax_region); + kref_init(&dax_region->kref); + dax_region->id = region_id; + dax_region->align = align; + dax_region->dev = parent; + dax_region->target_node = target_node; + ida_init(&dax_region->ida); + dax_region->res = (struct resource) { + .start = range->start, + .end = range->end, + .flags = IORESOURCE_MEM | flags, + }; + + rc = request_resource(&dax_regions, &dax_region->res); + if (rc) { + dev_err(parent, "dax_region resource conflict for %pR\n", + &dax_region->res); + goto err_res; + } + + if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) + goto err_sysfs; + + if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region)) + return NULL; + return dax_region; + +err_sysfs: + release_resource(&dax_region->res); +err_res: + dax_region_put(dax_region); + return NULL; +} +EXPORT_SYMBOL_GPL(alloc_dax_region); + +static void dax_mapping_release(struct device *dev) +{ + struct dax_mapping *mapping = to_dax_mapping(dev); + struct device *parent = dev->parent; + struct dev_dax *dev_dax = to_dev_dax(parent); + + ida_free(&dev_dax->ida, mapping->id); + kfree(mapping); + put_device(parent); +} + +static void unregister_dax_mapping(void *data) +{ + struct device *dev = data; + struct dax_mapping *mapping = to_dax_mapping(dev); + struct dev_dax *dev_dax = to_dev_dax(dev->parent); + + dev_dbg(dev, "%s\n", __func__); + + dev_dax->ranges[mapping->range_id].mapping = NULL; + mapping->range_id = -1; + + device_unregister(dev); +} + +static struct dev_dax_range *get_dax_range(struct device *dev) +{ + struct dax_mapping *mapping = to_dax_mapping(dev); + struct dev_dax *dev_dax = to_dev_dax(dev->parent); + int rc; + + rc = down_write_killable(&dax_region_rwsem); + if (rc) + return NULL; + if (mapping->range_id < 0) { + up_write(&dax_region_rwsem); + return NULL; + } + + return &dev_dax->ranges[mapping->range_id]; +} + +static void put_dax_range(void) +{ + up_write(&dax_region_rwsem); +} + +static ssize_t start_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax_range *dax_range; + ssize_t rc; + + dax_range = get_dax_range(dev); + if (!dax_range) + return -ENXIO; + rc = sysfs_emit(buf, "%#llx\n", dax_range->range.start); + put_dax_range(); + + return rc; +} +static DEVICE_ATTR(start, 0400, start_show, NULL); + +static ssize_t end_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax_range *dax_range; + ssize_t rc; + + dax_range = get_dax_range(dev); + if (!dax_range) + return -ENXIO; + rc = sysfs_emit(buf, "%#llx\n", dax_range->range.end); + put_dax_range(); + + return rc; +} +static DEVICE_ATTR(end, 0400, end_show, NULL); + +static ssize_t pgoff_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax_range *dax_range; + ssize_t rc; + + dax_range = get_dax_range(dev); + if (!dax_range) + return -ENXIO; + rc = sysfs_emit(buf, "%#lx\n", dax_range->pgoff); + put_dax_range(); + + return rc; +} +static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL); + +static struct attribute *dax_mapping_attributes[] = { + &dev_attr_start.attr, + &dev_attr_end.attr, + &dev_attr_page_offset.attr, + NULL, +}; + +static const struct attribute_group dax_mapping_attribute_group = { + .attrs = dax_mapping_attributes, +}; + +static const struct attribute_group *dax_mapping_attribute_groups[] = { + &dax_mapping_attribute_group, + NULL, +}; + +static const struct device_type dax_mapping_type = { + .release = dax_mapping_release, + .groups = dax_mapping_attribute_groups, +}; + +static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id) +{ + struct dax_region *dax_region = dev_dax->region; + struct dax_mapping *mapping; + struct device *dev; + int rc; + + lockdep_assert_held_write(&dax_region_rwsem); + + if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver, + "region disabled\n")) + return -ENXIO; + + mapping = kzalloc_obj(*mapping); + if (!mapping) + return -ENOMEM; + mapping->range_id = range_id; + mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL); + if (mapping->id < 0) { + kfree(mapping); + return -ENOMEM; + } + dev_dax->ranges[range_id].mapping = mapping; + dev = &mapping->dev; + device_initialize(dev); + dev->parent = &dev_dax->dev; + get_device(dev->parent); + dev->type = &dax_mapping_type; + dev_set_name(dev, "mapping%d", mapping->id); + rc = device_add(dev); + if (rc) { + put_device(dev); + return rc; + } + + rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping, + dev); + if (rc) + return rc; + return 0; +} + +static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start, + resource_size_t size) +{ + struct dax_region *dax_region = dev_dax->region; + struct resource *res = &dax_region->res; + struct device *dev = &dev_dax->dev; + struct dev_dax_range *ranges; + unsigned long pgoff = 0; + struct resource *alloc; + int i, rc; + + lockdep_assert_held_write(&dax_region_rwsem); + + /* handle the seed alloc special case */ + if (!size) { + if (dev_WARN_ONCE(dev, dev_dax->nr_range, + "0-size allocation must be first\n")) + return -EBUSY; + /* nr_range == 0 is elsewhere special cased as 0-size device */ + return 0; + } + + alloc = __request_region(res, start, size, dev_name(dev), 0); + if (!alloc) + return -ENOMEM; + + ranges = krealloc(dev_dax->ranges, sizeof(*ranges) + * (dev_dax->nr_range + 1), GFP_KERNEL); + if (!ranges) { + __release_region(res, alloc->start, resource_size(alloc)); + return -ENOMEM; + } + + for (i = 0; i < dev_dax->nr_range; i++) + pgoff += PHYS_PFN(range_len(&ranges[i].range)); + dev_dax->ranges = ranges; + ranges[dev_dax->nr_range++] = (struct dev_dax_range) { + .pgoff = pgoff, + .range = { + .start = alloc->start, + .end = alloc->end, + }, + }; + + dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1, + &alloc->start, &alloc->end); + /* + * A dev_dax instance must be registered before mapping device + * children can be added. Defer to devm_create_dev_dax() to add + * the initial mapping device. + */ + if (!device_is_registered(&dev_dax->dev)) + return 0; + + rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1); + if (rc) + trim_dev_dax_range(dev_dax); + + return rc; +} + +static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size) +{ + int last_range = dev_dax->nr_range - 1; + struct dev_dax_range *dax_range = &dev_dax->ranges[last_range]; + bool is_shrink = resource_size(res) > size; + struct range *range = &dax_range->range; + struct device *dev = &dev_dax->dev; + int rc; + + lockdep_assert_held_write(&dax_region_rwsem); + + if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n")) + return -EINVAL; + + rc = adjust_resource(res, range->start, size); + if (rc) + return rc; + + *range = (struct range) { + .start = range->start, + .end = range->start + size - 1, + }; + + dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend", + last_range, (unsigned long long) range->start, + (unsigned long long) range->end); + + return 0; +} + +static ssize_t size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + unsigned long long size; + int rc; + + rc = down_read_interruptible(&dax_dev_rwsem); + if (rc) + return rc; + size = dev_dax_size(dev_dax); + up_read(&dax_dev_rwsem); + + return sysfs_emit(buf, "%llu\n", size); +} + +static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size) +{ + /* + * The minimum mapping granularity for a device instance is a + * single subsection, unless the arch says otherwise. + */ + return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align())); +} + +static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size) +{ + resource_size_t to_shrink = dev_dax_size(dev_dax) - size; + struct dax_region *dax_region = dev_dax->region; + struct device *dev = &dev_dax->dev; + int i; + + for (i = dev_dax->nr_range - 1; i >= 0; i--) { + struct range *range = &dev_dax->ranges[i].range; + struct dax_mapping *mapping = dev_dax->ranges[i].mapping; + struct resource *adjust = NULL, *res; + resource_size_t shrink; + + shrink = min_t(u64, to_shrink, range_len(range)); + if (shrink >= range_len(range)) { + devm_release_action(dax_region->dev, + unregister_dax_mapping, &mapping->dev); + trim_dev_dax_range(dev_dax); + to_shrink -= shrink; + if (!to_shrink) + break; + continue; + } + + for_each_dax_region_resource(dax_region, res) + if (strcmp(res->name, dev_name(dev)) == 0 + && res->start == range->start) { + adjust = res; + break; + } + + if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1, + "failed to find matching resource\n")) + return -ENXIO; + return adjust_dev_dax_range(dev_dax, adjust, range_len(range) + - shrink); + } + return 0; +} + +/* + * Only allow adjustments that preserve the relative pgoff of existing + * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff. + */ +static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res) +{ + struct dev_dax_range *last; + int i; + + if (dev_dax->nr_range == 0) + return false; + if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0) + return false; + last = &dev_dax->ranges[dev_dax->nr_range - 1]; + if (last->range.start != res->start || last->range.end != res->end) + return false; + for (i = 0; i < dev_dax->nr_range - 1; i++) { + struct dev_dax_range *dax_range = &dev_dax->ranges[i]; + + if (dax_range->pgoff > last->pgoff) + return false; + } + + return true; +} + +static ssize_t dev_dax_resize(struct dax_region *dax_region, + struct dev_dax *dev_dax, resource_size_t size) +{ + resource_size_t avail = dax_region_avail_size(dax_region), to_alloc; + resource_size_t dev_size = dev_dax_size(dev_dax); + struct resource *region_res = &dax_region->res; + struct device *dev = &dev_dax->dev; + struct resource *res, *first; + resource_size_t alloc = 0; + int rc; + + if (dev->driver) + return -EBUSY; + if (size == dev_size) + return 0; + if (size > dev_size && size - dev_size > avail) + return -ENOSPC; + if (size < dev_size) + return dev_dax_shrink(dev_dax, size); + + to_alloc = size - dev_size; + if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc), + "resize of %pa misaligned\n", &to_alloc)) + return -ENXIO; + + /* + * Expand the device into the unused portion of the region. This + * may involve adjusting the end of an existing resource, or + * allocating a new resource. + */ +retry: + first = region_res->child; + if (!first) + return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc); + + rc = -ENOSPC; + for (res = first; res; res = res->sibling) { + struct resource *next = res->sibling; + + /* space at the beginning of the region */ + if (res == first && res->start > dax_region->res.start) { + alloc = min(res->start - dax_region->res.start, to_alloc); + rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc); + break; + } + + alloc = 0; + /* space between allocations */ + if (next && next->start > res->end + 1) + alloc = min(next->start - (res->end + 1), to_alloc); + + /* space at the end of the region */ + if (!alloc && !next && res->end < region_res->end) + alloc = min(region_res->end - res->end, to_alloc); + + if (!alloc) + continue; + + if (adjust_ok(dev_dax, res)) { + rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc); + break; + } + rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc); + break; + } + if (rc) + return rc; + to_alloc -= alloc; + if (to_alloc) + goto retry; + return 0; +} + +static ssize_t size_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + ssize_t rc; + unsigned long long val; + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_region *dax_region = dev_dax->region; + + rc = kstrtoull(buf, 0, &val); + if (rc) + return rc; + + if (!alloc_is_aligned(dev_dax, val)) { + dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val); + return -EINVAL; + } + + rc = down_write_killable(&dax_region_rwsem); + if (rc) + return rc; + if (!dax_region->dev->driver) { + rc = -ENXIO; + goto err_region; + } + rc = down_write_killable(&dax_dev_rwsem); + if (rc) + goto err_dev; + + rc = dev_dax_resize(dax_region, dev_dax, val); + +err_dev: + up_write(&dax_dev_rwsem); +err_region: + up_write(&dax_region_rwsem); + + if (rc == 0) + return len; + return rc; +} +static DEVICE_ATTR_RW(size); + +static ssize_t range_parse(const char *opt, size_t len, struct range *range) +{ + unsigned long long addr = 0; + char *start, *end, *str; + ssize_t rc = -EINVAL; + + str = kstrdup(opt, GFP_KERNEL); + if (!str) + return rc; + + end = str; + start = strsep(&end, "-"); + if (!start || !end) + goto err; + + rc = kstrtoull(start, 16, &addr); + if (rc) + goto err; + range->start = addr; + + rc = kstrtoull(end, 16, &addr); + if (rc) + goto err; + range->end = addr; + +err: + kfree(str); + return rc; +} + +static ssize_t mapping_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_region *dax_region = dev_dax->region; + size_t to_alloc; + struct range r; + ssize_t rc; + + rc = range_parse(buf, len, &r); + if (rc) + return rc; + + rc = down_write_killable(&dax_region_rwsem); + if (rc) + return rc; + if (!dax_region->dev->driver) { + up_write(&dax_region_rwsem); + return rc; + } + rc = down_write_killable(&dax_dev_rwsem); + if (rc) { + up_write(&dax_region_rwsem); + return rc; + } + + to_alloc = range_len(&r); + if (alloc_is_aligned(dev_dax, to_alloc)) + rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc); + up_write(&dax_dev_rwsem); + up_write(&dax_region_rwsem); + + return rc == 0 ? len : rc; +} +static DEVICE_ATTR_WO(mapping); + +static ssize_t align_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + + return sysfs_emit(buf, "%d\n", dev_dax->align); +} + +static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax) +{ + struct device *dev = &dev_dax->dev; + int i; + + for (i = 0; i < dev_dax->nr_range; i++) { + size_t len = range_len(&dev_dax->ranges[i].range); + + if (!alloc_is_aligned(dev_dax, len)) { + dev_dbg(dev, "%s: align %u invalid for range %d\n", + __func__, dev_dax->align, i); + return -EINVAL; + } + } + + return 0; +} + +static ssize_t align_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_region *dax_region = dev_dax->region; + unsigned long val, align_save; + ssize_t rc; + + rc = kstrtoul(buf, 0, &val); + if (rc) + return -ENXIO; + + if (!dax_align_valid(val)) + return -EINVAL; + + rc = down_write_killable(&dax_region_rwsem); + if (rc) + return rc; + if (!dax_region->dev->driver) { + up_write(&dax_region_rwsem); + return -ENXIO; + } + + rc = down_write_killable(&dax_dev_rwsem); + if (rc) { + up_write(&dax_region_rwsem); + return rc; + } + if (dev->driver) { + rc = -EBUSY; + goto out_unlock; + } + + align_save = dev_dax->align; + dev_dax->align = val; + rc = dev_dax_validate_align(dev_dax); + if (rc) + dev_dax->align = align_save; +out_unlock: + up_write(&dax_dev_rwsem); + up_write(&dax_region_rwsem); + return rc == 0 ? len : rc; +} +static DEVICE_ATTR_RW(align); + +static int dev_dax_target_node(struct dev_dax *dev_dax) +{ + struct dax_region *dax_region = dev_dax->region; + + return dax_region->target_node; +} + +static ssize_t target_node_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + + return sysfs_emit(buf, "%d\n", dev_dax_target_node(dev_dax)); +} +static DEVICE_ATTR_RO(target_node); + +static ssize_t resource_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_region *dax_region = dev_dax->region; + unsigned long long start; + + if (dev_dax->nr_range < 1) + start = dax_region->res.start; + else + start = dev_dax->ranges[0].range.start; + + return sysfs_emit(buf, "%#llx\n", start); +} +static DEVICE_ATTR(resource, 0400, resource_show, NULL); + +static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + /* + * We only ever expect to handle device-dax instances, i.e. the + * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero + */ + return sysfs_emit(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0); +} +static DEVICE_ATTR_RO(modalias); + +static ssize_t numa_node_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", dev_to_node(dev)); +} +static DEVICE_ATTR_RO(numa_node); + +static ssize_t memmap_on_memory_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + + return sysfs_emit(buf, "%d\n", dev_dax->memmap_on_memory); +} + +static ssize_t memmap_on_memory_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + bool val; + int rc; + + rc = kstrtobool(buf, &val); + if (rc) + return rc; + + if (val == true && !mhp_supports_memmap_on_memory()) { + dev_dbg(dev, "memmap_on_memory is not available\n"); + return -EOPNOTSUPP; + } + + rc = down_write_killable(&dax_dev_rwsem); + if (rc) + return rc; + + if (dev_dax->memmap_on_memory != val && dev->driver && + to_dax_drv(dev->driver)->type == DAXDRV_KMEM_TYPE) { + up_write(&dax_dev_rwsem); + return -EBUSY; + } + + dev_dax->memmap_on_memory = val; + up_write(&dax_dev_rwsem); + + return len; +} +static DEVICE_ATTR_RW(memmap_on_memory); + +static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_region *dax_region = dev_dax->region; + + if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0) + return 0; + if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA)) + return 0; + if (a == &dev_attr_mapping.attr && is_static(dax_region)) + return 0; + if ((a == &dev_attr_align.attr || + a == &dev_attr_size.attr) && is_static(dax_region)) + return 0444; + return a->mode; +} + +static struct attribute *dev_dax_attributes[] = { + &dev_attr_modalias.attr, + &dev_attr_size.attr, + &dev_attr_mapping.attr, + &dev_attr_target_node.attr, + &dev_attr_align.attr, + &dev_attr_resource.attr, + &dev_attr_numa_node.attr, + &dev_attr_memmap_on_memory.attr, + NULL, +}; + +static const struct attribute_group dev_dax_attribute_group = { + .attrs = dev_dax_attributes, + .is_visible = dev_dax_visible, +}; + +static const struct attribute_group *dax_attribute_groups[] = { + &dev_dax_attribute_group, + NULL, +}; + +static void dev_dax_release(struct device *dev) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_device *dax_dev = dev_dax->dax_dev; + + put_dax(dax_dev); + free_dev_dax_id(dev_dax); + kfree(dev_dax->pgmap); + kfree(dev_dax); +} + +static const struct device_type dev_dax_type = { + .release = dev_dax_release, + .groups = dax_attribute_groups, +}; + +/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */ +__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, + unsigned long size) +{ + for (int i = 0; i < dev_dax->nr_range; i++) { + struct dev_dax_range *dax_range = &dev_dax->ranges[i]; + struct range *range = &dax_range->range; + phys_addr_t phys; + + if (!in_range(pgoff, dax_range->pgoff, PHYS_PFN(range_len(range)))) + continue; + phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start; + if (phys + size - 1 <= range->end) + return phys; + break; + } + return -1; +} +EXPORT_SYMBOL_GPL(dax_pgoff_to_phys); + +static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data) +{ + struct dax_region *dax_region = data->dax_region; + struct device *parent = dax_region->dev; + struct dax_device *dax_dev; + struct dev_dax *dev_dax; + struct inode *inode; + struct device *dev; + int rc; + + dev_dax = kzalloc_obj(*dev_dax); + if (!dev_dax) + return ERR_PTR(-ENOMEM); + + dev_dax->region = dax_region; + if (is_static(dax_region)) { + if (dev_WARN_ONCE(parent, data->id < 0, + "dynamic id specified to static region\n")) { + rc = -EINVAL; + goto err_id; + } + + dev_dax->id = data->id; + } else { + if (dev_WARN_ONCE(parent, data->id >= 0, + "static id specified to dynamic region\n")) { + rc = -EINVAL; + goto err_id; + } + + rc = alloc_dev_dax_id(dev_dax); + if (rc < 0) + goto err_id; + } + + dev = &dev_dax->dev; + device_initialize(dev); + dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id); + + rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size); + if (rc) + goto err_range; + + if (data->pgmap) { + dev_WARN_ONCE(parent, !is_static(dax_region), + "custom dev_pagemap requires a static dax_region\n"); + + dev_dax->pgmap = kmemdup(data->pgmap, + sizeof(struct dev_pagemap), GFP_KERNEL); + if (!dev_dax->pgmap) { + rc = -ENOMEM; + goto err_pgmap; + } + } + + /* + * No dax_operations since there is no access to this device outside of + * mmap of the resulting character device. + */ + dax_dev = alloc_dax(dev_dax, NULL); + if (IS_ERR(dax_dev)) { + rc = PTR_ERR(dax_dev); + goto err_alloc_dax; + } + set_dax_synchronous(dax_dev); + set_dax_nocache(dax_dev); + set_dax_nomc(dax_dev); + + /* a device_dax instance is dead while the driver is not attached */ + kill_dax(dax_dev); + + dev_dax->dax_dev = dax_dev; + dev_dax->target_node = dax_region->target_node; + dev_dax->align = dax_region->align; + ida_init(&dev_dax->ida); + + dev_dax->memmap_on_memory = data->memmap_on_memory; + + inode = dax_inode(dax_dev); + dev->devt = inode->i_rdev; + dev->bus = &dax_bus_type; + dev->parent = parent; + dev->type = &dev_dax_type; + + rc = device_add(dev); + if (rc) { + kill_dev_dax(dev_dax); + put_device(dev); + return ERR_PTR(rc); + } + + rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev); + if (rc) + return ERR_PTR(rc); + + /* register mapping device for the initial allocation range */ + if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) { + rc = devm_register_dax_mapping(dev_dax, 0); + if (rc) + return ERR_PTR(rc); + } + + return dev_dax; + +err_alloc_dax: + kfree(dev_dax->pgmap); +err_pgmap: + free_dev_dax_ranges(dev_dax); +err_range: + free_dev_dax_id(dev_dax); +err_id: + kfree(dev_dax); + + return ERR_PTR(rc); +} + +struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data) +{ + struct dev_dax *dev_dax; + + down_write(&dax_region_rwsem); + dev_dax = __devm_create_dev_dax(data); + up_write(&dax_region_rwsem); + + return dev_dax; +} +EXPORT_SYMBOL_GPL(devm_create_dev_dax); + +int __dax_driver_register(struct dax_device_driver *dax_drv, + struct module *module, const char *mod_name) +{ + struct device_driver *drv = &dax_drv->drv; + + /* + * dax_bus_probe() calls dax_drv->probe() unconditionally. + * So better be safe than sorry and ensure it is provided. + */ + if (!dax_drv->probe) + return -EINVAL; + + INIT_LIST_HEAD(&dax_drv->ids); + drv->owner = module; + drv->name = mod_name; + drv->mod_name = mod_name; + drv->bus = &dax_bus_type; + + return driver_register(drv); +} +EXPORT_SYMBOL_GPL(__dax_driver_register); + +void dax_driver_unregister(struct dax_device_driver *dax_drv) +{ + struct device_driver *drv = &dax_drv->drv; + struct dax_id *dax_id, *_id; + + mutex_lock(&dax_bus_lock); + list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) { + list_del(&dax_id->list); + kfree(dax_id); + } + mutex_unlock(&dax_bus_lock); + driver_unregister(drv); +} +EXPORT_SYMBOL_GPL(dax_driver_unregister); + +int __init dax_bus_init(void) +{ + return bus_register(&dax_bus_type); +} + +void __exit dax_bus_exit(void) +{ + bus_unregister(&dax_bus_type); +} diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h new file mode 100644 index 000000000..5909171a4 --- /dev/null +++ b/drivers/dax/bus.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */ +#ifndef __DAX_BUS_H__ +#define __DAX_BUS_H__ +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/range.h> +#include <linux/workqueue.h> + +struct dev_dax; +struct resource; +struct dax_device; +struct dax_region; + +/* dax bus specific ioresource flags */ +#define IORESOURCE_DAX_STATIC BIT(0) +#define IORESOURCE_DAX_KMEM BIT(1) + +struct dax_region *alloc_dax_region(struct device *parent, int region_id, + struct range *range, int target_node, unsigned int align, + unsigned long flags); + +struct dev_dax_data { + struct dax_region *dax_region; + struct dev_pagemap *pgmap; + resource_size_t size; + int id; + bool memmap_on_memory; +}; + +struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data); + +enum dax_driver_type { + DAXDRV_KMEM_TYPE, + DAXDRV_DEVICE_TYPE, + DAXDRV_FSDEV_TYPE, +}; + +struct dax_device_driver { + struct device_driver drv; + struct list_head ids; + enum dax_driver_type type; + int (*probe)(struct dev_dax *dev); + void (*remove)(struct dev_dax *dev); +}; + +#define to_dax_drv(__drv) container_of_const(__drv, struct dax_device_driver, drv) + +int __dax_driver_register(struct dax_device_driver *dax_drv, + struct module *module, const char *mod_name); +#define dax_driver_register(driver) \ + __dax_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) +void dax_driver_unregister(struct dax_device_driver *dax_drv); +void kill_dev_dax(struct dev_dax *dev_dax); +bool static_dev_dax(struct dev_dax *dev_dax); + +struct hmem_platform_device { + struct platform_device pdev; + struct work_struct work; + bool did_probe; +}; + +static inline struct hmem_platform_device * +to_hmem_platform_device(struct platform_device *pdev) +{ + return container_of(pdev, struct hmem_platform_device, pdev); +} + +#if IS_ENABLED(CONFIG_DEV_DAX_HMEM) +void dax_hmem_flush_work(void); +#else +static inline void dax_hmem_flush_work(void) { } +#endif + +#define MODULE_ALIAS_DAX_DEVICE(type) \ + MODULE_ALIAS("dax:t" __stringify(type) "*") +#define DAX_DEVICE_MODALIAS_FMT "dax:t%d" + +#endif /* __DAX_BUS_H__ */ diff --git a/drivers/dax/cxl.c b/drivers/dax/cxl.c new file mode 100644 index 000000000..3ab39b778 --- /dev/null +++ b/drivers/dax/cxl.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation. All rights reserved. */ +#include <linux/module.h> +#include <linux/dax.h> + +#include "../cxl/cxl.h" +#include "bus.h" + +static int cxl_dax_region_probe(struct device *dev) +{ + struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev); + int nid = phys_to_target_node(cxlr_dax->hpa_range.start); + struct cxl_region *cxlr = cxlr_dax->cxlr; + struct dax_region *dax_region; + struct dev_dax_data data; + + if (nid == NUMA_NO_NODE) + nid = memory_add_physaddr_to_nid(cxlr_dax->hpa_range.start); + + dax_region = alloc_dax_region(dev, cxlr->id, &cxlr_dax->hpa_range, nid, + PMD_SIZE, IORESOURCE_DAX_KMEM); + if (!dax_region) + return -ENOMEM; + + data = (struct dev_dax_data) { + .dax_region = dax_region, + .id = -1, + .size = range_len(&cxlr_dax->hpa_range), + .memmap_on_memory = true, + }; + + return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data)); +} + +static struct cxl_driver cxl_dax_region_driver = { + .name = "cxl_dax_region", + .probe = cxl_dax_region_probe, + .id = CXL_DEVICE_DAX_REGION, + .drv = { + .suppress_bind_attrs = true, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +static void cxl_dax_region_driver_register(struct work_struct *work) +{ + dax_hmem_flush_work(); + cxl_driver_register(&cxl_dax_region_driver); +} + +static DECLARE_WORK(cxl_dax_region_driver_work, cxl_dax_region_driver_register); + +static int __init cxl_dax_region_init(void) +{ + /* + * Need to resolve a race with dax_hmem wanting to drive regions + * instead of CXL + */ + queue_work(system_long_wq, &cxl_dax_region_driver_work); + return 0; +} +module_init(cxl_dax_region_init); + +static void __exit cxl_dax_region_exit(void) +{ + flush_work(&cxl_dax_region_driver_work); + cxl_driver_unregister(&cxl_dax_region_driver); +} +module_exit(cxl_dax_region_exit); + +MODULE_ALIAS_CXL(CXL_DEVICE_DAX_REGION); +MODULE_DESCRIPTION("CXL DAX: direct access to CXL regions"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_IMPORT_NS("CXL"); diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h new file mode 100644 index 000000000..81e4af49e --- /dev/null +++ b/drivers/dax/dax-private.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright(c) 2016 Intel Corporation. All rights reserved. + */ +#ifndef __DAX_PRIVATE_H__ +#define __DAX_PRIVATE_H__ + +#include <linux/device.h> +#include <linux/cdev.h> +#include <linux/idr.h> + +/* private routines between core files */ +struct dax_device; +struct dax_device *inode_dax(struct inode *inode); +struct inode *dax_inode(struct dax_device *dax_dev); +int dax_bus_init(void); +void dax_bus_exit(void); + +/** + * struct dax_region - mapping infrastructure for dax devices + * @id: kernel-wide unique region for a memory range + * @target_node: effective numa node if this memory range is onlined + * @kref: to pin while other agents have a need to do lookups + * @dev: parent device backing this region + * @align: allocation and mapping alignment for child dax devices + * @ida: instance id allocator + * @res: resource tree to track instance allocations + * @seed: allow userspace to find the first unbound seed device + * @youngest: allow userspace to find the most recently created device + */ +struct dax_region { + int id; + int target_node; + struct kref kref; + struct device *dev; + unsigned int align; + struct ida ida; + struct resource res; + struct device *seed; + struct device *youngest; +}; + +/** + * struct dax_mapping - device to display mapping range attributes + * @dev: device representing this range + * @range_id: index within dev_dax ranges array + * @id: ida of this mapping + */ +struct dax_mapping { + struct device dev; + int range_id; + int id; +}; + +/** + * struct dev_dax_range - tuple represenging a range of memory used by dev_dax + * @pgoff: page offset + * @range: resource-span + * @mapping: reference to the dax_mapping for this range + */ +struct dev_dax_range { + unsigned long pgoff; + struct range range; + struct dax_mapping *mapping; +}; + +/** + * struct dev_dax - instance data for a subdivision of a dax region, and + * data while the device is activated in the driver. + * @region: parent region + * @dax_dev: core dax functionality + * @virt_addr: kva from memremap; used by fsdev_dax + * @cached_size: size of daxdev cached by fsdev_dax + * @align: alignment of this instance + * @target_node: effective numa node if dev_dax memory range is onlined + * @dyn_id: is this a dynamic or statically created instance + * @id: ida allocated id when the dax_region is not static + * @ida: mapping id allocator + * @dev: device core + * @pgmap: pgmap for memmap setup / lifetime (driver owned) + * @memmap_on_memory: allow kmem to put the memmap in the memory + * @nr_range: size of @ranges + * @ranges: range tuples of memory used + */ +struct dev_dax { + struct dax_region *region; + struct dax_device *dax_dev; + void *virt_addr; + u64 cached_size; + unsigned int align; + int target_node; + bool dyn_id; + int id; + struct ida ida; + struct device dev; + struct dev_pagemap *pgmap; + bool memmap_on_memory; + int nr_range; + struct dev_dax_range *ranges; +}; + +/* + * While run_dax() is potentially a generic operation that could be + * defined in include/linux/dax.h we don't want to grow any users + * outside of drivers/dax/ + */ +void run_dax(struct dax_device *dax_dev); + +static inline struct dev_dax *to_dev_dax(struct device *dev) +{ + return container_of(dev, struct dev_dax, dev); +} + +static inline struct dax_mapping *to_dax_mapping(struct device *dev) +{ + return container_of(dev, struct dax_mapping, dev); +} + +phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, unsigned long size); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline bool dax_align_valid(unsigned long align) +{ + if (align == PUD_SIZE && IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) + return true; + if (align == PMD_SIZE && has_transparent_hugepage()) + return true; + if (align == PAGE_SIZE) + return true; + return false; +} +#else +static inline bool dax_align_valid(unsigned long align) +{ + return align == PAGE_SIZE; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif diff --git a/drivers/dax/device.c b/drivers/dax/device.c new file mode 100644 index 000000000..d0c9b4e03 --- /dev/null +++ b/drivers/dax/device.c @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> +#include <linux/pagemap.h> +#include <linux/module.h> +#include <linux/device.h> +#include <linux/cdev.h> +#include <linux/slab.h> +#include <linux/dax.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include "dax-private.h" +#include "bus.h" + +static int __check_vma(struct dev_dax *dev_dax, vma_flags_t flags, + unsigned long start, unsigned long end, struct file *file, + const char *func) +{ + struct device *dev = &dev_dax->dev; + unsigned long mask; + + if (!dax_alive(dev_dax->dax_dev)) + return -ENXIO; + + /* prevent private mappings from being established */ + if (!vma_flags_test_any(&flags, VMA_MAYSHARE_BIT)) { + dev_info_ratelimited(dev, + "%s: %s: fail, attempted private mapping\n", + current->comm, func); + return -EINVAL; + } + + mask = dev_dax->align - 1; + if (start & mask || end & mask) { + dev_info_ratelimited(dev, + "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", + current->comm, func, start, end, + mask); + return -EINVAL; + } + + if (!file_is_dax(file)) { + dev_info_ratelimited(dev, + "%s: %s: fail, vma is not DAX capable\n", + current->comm, func); + return -EINVAL; + } + + return 0; +} + +static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, + const char *func) +{ + return __check_vma(dev_dax, vma->flags, vma->vm_start, vma->vm_end, + vma->vm_file, func); +} + +static void dax_set_mapping(struct vm_fault *vmf, unsigned long pfn, + unsigned long fault_size) +{ + unsigned long i, nr_pages = fault_size / PAGE_SIZE; + struct file *filp = vmf->vma->vm_file; + struct dev_dax *dev_dax = filp->private_data; + pgoff_t pgoff; + + /* mapping is only set on the head */ + if (dev_dax->pgmap->vmemmap_shift) + nr_pages = 1; + + pgoff = linear_page_index(vmf->vma, + ALIGN_DOWN(vmf->address, fault_size)); + + for (i = 0; i < nr_pages; i++) { + struct folio *folio = pfn_folio(pfn + i); + + if (folio->mapping) + continue; + + folio->mapping = filp->f_mapping; + folio->index = pgoff + i; + } +} + +static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf) +{ + struct device *dev = &dev_dax->dev; + phys_addr_t phys; + unsigned long pfn; + unsigned int fault_size = PAGE_SIZE; + + if (check_vma(dev_dax, vmf->vma, __func__)) + return VM_FAULT_SIGBUS; + + if (dev_dax->align > PAGE_SIZE) { + dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", + dev_dax->align, fault_size); + return VM_FAULT_SIGBUS; + } + + if (fault_size != dev_dax->align) + return VM_FAULT_SIGBUS; + + phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE); + if (phys == -1) { + dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff); + return VM_FAULT_SIGBUS; + } + + pfn = PHYS_PFN(phys); + + dax_set_mapping(vmf, pfn, fault_size); + + return vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn), + vmf->flags & FAULT_FLAG_WRITE); +} + +static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf) +{ + unsigned long pmd_addr = vmf->address & PMD_MASK; + struct device *dev = &dev_dax->dev; + phys_addr_t phys; + pgoff_t pgoff; + unsigned long pfn; + unsigned int fault_size = PMD_SIZE; + + if (check_vma(dev_dax, vmf->vma, __func__)) + return VM_FAULT_SIGBUS; + + if (dev_dax->align > PMD_SIZE) { + dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", + dev_dax->align, fault_size); + return VM_FAULT_SIGBUS; + } + + if (fault_size < dev_dax->align) + return VM_FAULT_SIGBUS; + else if (fault_size > dev_dax->align) + return VM_FAULT_FALLBACK; + + /* if we are outside of the VMA */ + if (pmd_addr < vmf->vma->vm_start || + (pmd_addr + PMD_SIZE) > vmf->vma->vm_end) + return VM_FAULT_SIGBUS; + + pgoff = linear_page_index(vmf->vma, pmd_addr); + phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE); + if (phys == -1) { + dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); + return VM_FAULT_SIGBUS; + } + + pfn = PHYS_PFN(phys); + + dax_set_mapping(vmf, pfn, fault_size); + + return vmf_insert_folio_pmd(vmf, page_folio(pfn_to_page(pfn)), + vmf->flags & FAULT_FLAG_WRITE); +} + +#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD +static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf) +{ + unsigned long pud_addr = vmf->address & PUD_MASK; + struct device *dev = &dev_dax->dev; + phys_addr_t phys; + pgoff_t pgoff; + unsigned long pfn; + unsigned int fault_size = PUD_SIZE; + + + if (check_vma(dev_dax, vmf->vma, __func__)) + return VM_FAULT_SIGBUS; + + if (dev_dax->align > PUD_SIZE) { + dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", + dev_dax->align, fault_size); + return VM_FAULT_SIGBUS; + } + + if (fault_size < dev_dax->align) + return VM_FAULT_SIGBUS; + else if (fault_size > dev_dax->align) + return VM_FAULT_FALLBACK; + + /* if we are outside of the VMA */ + if (pud_addr < vmf->vma->vm_start || + (pud_addr + PUD_SIZE) > vmf->vma->vm_end) + return VM_FAULT_SIGBUS; + + pgoff = linear_page_index(vmf->vma, pud_addr); + phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE); + if (phys == -1) { + dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); + return VM_FAULT_SIGBUS; + } + + pfn = PHYS_PFN(phys); + + dax_set_mapping(vmf, pfn, fault_size); + + return vmf_insert_folio_pud(vmf, page_folio(pfn_to_page(pfn)), + vmf->flags & FAULT_FLAG_WRITE); +} +#else +static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf) +{ + return VM_FAULT_FALLBACK; +} +#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ + +static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, unsigned int order) +{ + struct file *filp = vmf->vma->vm_file; + vm_fault_t rc = VM_FAULT_SIGBUS; + int id; + struct dev_dax *dev_dax = filp->private_data; + + dev_dbg(&dev_dax->dev, "%s: op=%s addr=%#lx order=%d\n", current->comm, + (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read", + vmf->address & ~((1UL << (order + PAGE_SHIFT)) - 1), order); + + id = dax_read_lock(); + if (order == 0) + rc = __dev_dax_pte_fault(dev_dax, vmf); + else if (order == PMD_ORDER) + rc = __dev_dax_pmd_fault(dev_dax, vmf); + else if (order == PUD_ORDER) + rc = __dev_dax_pud_fault(dev_dax, vmf); + else + rc = VM_FAULT_SIGBUS; + + dax_read_unlock(id); + + return rc; +} + +static vm_fault_t dev_dax_fault(struct vm_fault *vmf) +{ + return dev_dax_huge_fault(vmf, 0); +} + +static int dev_dax_may_split(struct vm_area_struct *vma, unsigned long addr) +{ + struct file *filp = vma->vm_file; + struct dev_dax *dev_dax = filp->private_data; + + if (!IS_ALIGNED(addr, dev_dax->align)) + return -EINVAL; + return 0; +} + +static unsigned long dev_dax_pagesize(struct vm_area_struct *vma) +{ + struct file *filp = vma->vm_file; + struct dev_dax *dev_dax = filp->private_data; + + return dev_dax->align; +} + +static const struct vm_operations_struct dax_vm_ops = { + .fault = dev_dax_fault, + .huge_fault = dev_dax_huge_fault, + .may_split = dev_dax_may_split, + .pagesize = dev_dax_pagesize, +}; + +static int dax_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *filp = desc->file; + struct dev_dax *dev_dax = filp->private_data; + int rc, id; + + dev_dbg(&dev_dax->dev, "trace\n"); + + /* + * We lock to check dax_dev liveness and will re-check at + * fault time. + */ + id = dax_read_lock(); + rc = __check_vma(dev_dax, desc->vma_flags, desc->start, desc->end, filp, + __func__); + dax_read_unlock(id); + if (rc) + return rc; + + desc->vm_ops = &dax_vm_ops; + vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT); + return 0; +} + +/* return an unmapped area aligned to the dax region specified alignment */ +static unsigned long dax_get_unmapped_area(struct file *filp, + unsigned long addr, unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + unsigned long off, off_end, off_align, len_align, addr_align, align; + struct dev_dax *dev_dax = filp ? filp->private_data : NULL; + + if (!dev_dax || addr) + goto out; + + align = dev_dax->align; + off = pgoff << PAGE_SHIFT; + off_end = off + len; + off_align = round_up(off, align); + + if ((off_end <= off_align) || ((off_end - off_align) < align)) + goto out; + + len_align = len + align; + if ((off + len_align) < off) + goto out; + + addr_align = mm_get_unmapped_area(filp, addr, len_align, pgoff, flags); + if (!IS_ERR_VALUE(addr_align)) { + addr_align += (off - addr_align) & (align - 1); + return addr_align; + } + out: + return mm_get_unmapped_area(filp, addr, len, pgoff, flags); +} + +static const struct address_space_operations dev_dax_aops = { + .dirty_folio = noop_dirty_folio, +}; + +static int dax_open(struct inode *inode, struct file *filp) +{ + struct dax_device *dax_dev = inode_dax(inode); + struct inode *__dax_inode = dax_inode(dax_dev); + struct dev_dax *dev_dax = dax_get_private(dax_dev); + + dev_dbg(&dev_dax->dev, "trace\n"); + inode->i_mapping = __dax_inode->i_mapping; + inode->i_mapping->host = __dax_inode; + inode->i_mapping->a_ops = &dev_dax_aops; + filp->f_mapping = inode->i_mapping; + filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); + filp->f_sb_err = file_sample_sb_err(filp); + filp->private_data = dev_dax; + inode->i_flags = S_DAX; + + return 0; +} + +static int dax_release(struct inode *inode, struct file *filp) +{ + struct dev_dax *dev_dax = filp->private_data; + + dev_dbg(&dev_dax->dev, "trace\n"); + return 0; +} + +static const struct file_operations dax_fops = { + .llseek = noop_llseek, + .owner = THIS_MODULE, + .open = dax_open, + .release = dax_release, + .get_unmapped_area = dax_get_unmapped_area, + .mmap_prepare = dax_mmap_prepare, + .fop_flags = FOP_MMAP_SYNC, +}; + +static void dev_dax_cdev_del(void *cdev) +{ + cdev_del(cdev); +} + +static void dev_dax_kill(void *dev_dax) +{ + kill_dev_dax(dev_dax); +} + +static int dev_dax_probe(struct dev_dax *dev_dax) +{ + struct dax_device *dax_dev = dev_dax->dax_dev; + struct device *dev = &dev_dax->dev; + struct dev_pagemap *pgmap; + struct inode *inode; + struct cdev *cdev; + void *addr; + int rc, i; + + if (static_dev_dax(dev_dax)) { + if (dev_dax->nr_range > 1) { + dev_warn(dev, + "static pgmap / multi-range device conflict\n"); + return -EINVAL; + } + + pgmap = dev_dax->pgmap; + } else { + if (dev_dax->pgmap) { + dev_warn(dev, + "dynamic-dax with pre-populated page map\n"); + return -EINVAL; + } + + pgmap = devm_kzalloc(dev, + struct_size(pgmap, ranges, dev_dax->nr_range - 1), + GFP_KERNEL); + if (!pgmap) + return -ENOMEM; + + pgmap->nr_range = dev_dax->nr_range; + dev_dax->pgmap = pgmap; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + pgmap->ranges[i] = *range; + } + } + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + + if (!devm_request_mem_region(dev, range->start, + range_len(range), dev_name(dev))) { + dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", + i, range->start, range->end); + return -EBUSY; + } + } + + pgmap->type = MEMORY_DEVICE_GENERIC; + if (dev_dax->align > PAGE_SIZE) + pgmap->vmemmap_shift = + order_base_2(dev_dax->align >> PAGE_SHIFT); + addr = devm_memremap_pages(dev, pgmap); + if (IS_ERR(addr)) + return PTR_ERR(addr); + + inode = dax_inode(dax_dev); + cdev = inode->i_cdev; + cdev_init(cdev, &dax_fops); + cdev->owner = dev->driver->owner; + cdev_set_parent(cdev, &dev->kobj); + rc = cdev_add(cdev, dev->devt, 1); + if (rc) + return rc; + + rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev); + if (rc) + return rc; + + run_dax(dax_dev); + return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax); +} + +static struct dax_device_driver device_dax_driver = { + .probe = dev_dax_probe, + .type = DAXDRV_DEVICE_TYPE, +}; + +static int __init dax_init(void) +{ + return dax_driver_register(&device_dax_driver); +} + +static void __exit dax_exit(void) +{ + dax_driver_unregister(&device_dax_driver); +} + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("Device DAX: direct access device driver"); +MODULE_LICENSE("GPL v2"); +module_init(dax_init); +module_exit(dax_exit); +MODULE_ALIAS_DAX_DEVICE(0); diff --git a/drivers/dax/fsdev.c b/drivers/dax/fsdev.c new file mode 100644 index 000000000..188b2526b --- /dev/null +++ b/drivers/dax/fsdev.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2026 Micron Technology, Inc. */ +#include <linux/memremap.h> +#include <linux/pagemap.h> +#include <linux/module.h> +#include <linux/device.h> +#include <linux/cdev.h> +#include <linux/slab.h> +#include <linux/dax.h> +#include <linux/uio.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include "dax-private.h" +#include "bus.h" + +/* + * FS-DAX compatible devdax driver + * + * Unlike drivers/dax/device.c which pre-initializes compound folios based + * on device alignment (via vmemmap_shift), this driver leaves folios + * uninitialized similar to pmem. This allows fs-dax filesystems like famfs + * to work without needing special handling for pre-initialized folios. + * + * Key differences from device.c: + * - pgmap type is MEMORY_DEVICE_FS_DAX (not MEMORY_DEVICE_GENERIC) + * - vmemmap_shift is NOT set (folios remain order-0) + * - fs-dax can dynamically create compound folios as needed + * - No mmap support - all access is through fs-dax/iomap + */ + +static void fsdev_write_dax(void *addr, struct page *page, + unsigned int off, unsigned int len) +{ + while (len) { + void *mem = kmap_local_page(page); + unsigned int chunk = min_t(unsigned int, len, PAGE_SIZE - off); + + memcpy_flushcache(addr, mem + off, chunk); + kunmap_local(mem); + len -= chunk; + off = 0; + page++; + addr += chunk; + } +} + +static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, + long nr_pages, enum dax_access_mode mode, void **kaddr, + unsigned long *pfn) +{ + struct dev_dax *dev_dax = dax_get_private(dax_dev); + size_t size = nr_pages << PAGE_SHIFT; + size_t offset = pgoff << PAGE_SHIFT; + void *virt_addr = dev_dax->virt_addr + offset; + phys_addr_t phys; + unsigned long local_pfn; + + phys = dax_pgoff_to_phys(dev_dax, pgoff, size); + if (phys == -1) { + dev_dbg(&dev_dax->dev, + "pgoff (%#lx) out of range\n", pgoff); + return -EFAULT; + } + + if (kaddr) + *kaddr = virt_addr; + + local_pfn = PHYS_PFN(phys); + if (pfn) + *pfn = local_pfn; + + /* + * Use cached_size which was computed at probe time. The size cannot + * change while the driver is bound (resize returns -EBUSY). + */ + return PHYS_PFN(min(size, dev_dax->cached_size - offset)); +} + +static int fsdev_dax_zero_page_range(struct dax_device *dax_dev, + pgoff_t pgoff, size_t nr_pages) +{ + void *kaddr; + long rc; + + WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__); + rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, &kaddr, NULL); + if (rc < 0) + return rc; + fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE); + return 0; +} + +static long fsdev_dax_direct_access(struct dax_device *dax_dev, + pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, + void **kaddr, unsigned long *pfn) +{ + return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode, + kaddr, pfn); +} + +static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, + void *addr, size_t bytes, struct iov_iter *i) +{ + return _copy_from_iter_flushcache(addr, bytes, i); +} + +static const struct dax_operations dev_dax_ops = { + .direct_access = fsdev_dax_direct_access, + .zero_page_range = fsdev_dax_zero_page_range, + .recovery_write = fsdev_dax_recovery_write, +}; + +static void fsdev_cdev_del(void *cdev) +{ + cdev_del(cdev); +} + +static void fsdev_kill(void *dev_dax) +{ + kill_dev_dax(dev_dax); +} + +static void fsdev_clear_ops(void *data) +{ + struct dev_dax *dev_dax = data; + + dax_set_ops(dev_dax->dax_dev, NULL); +} + +/* + * Page map operations for FS-DAX mode + * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c + * + * Note: folio_free callback is not needed for MEMORY_DEVICE_FS_DAX. + * The core mm code in free_zone_device_folio() handles the wake_up_var() + * directly for this memory type. + */ +static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap, + unsigned long pfn, unsigned long nr_pages, int mf_flags) +{ + struct dev_dax *dev_dax = pgmap->owner; + u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start; + u64 len = nr_pages << PAGE_SHIFT; + + return dax_holder_notify_failure(dev_dax->dax_dev, offset, + len, mf_flags); +} + +static const struct dev_pagemap_ops fsdev_pagemap_ops = { + .memory_failure = fsdev_pagemap_memory_failure, +}; + +/* + * Clear any stale folio state from pages in the given range. + * This is necessary because device_dax pre-initializes compound folios + * based on vmemmap_shift, and that state may persist after driver unbind. + * Since fsdev_dax uses MEMORY_DEVICE_FS_DAX without vmemmap_shift, fs-dax + * expects to find clean order-0 folios that it can build into compound + * folios on demand. + * + * At probe time, no filesystem should be mounted yet, so all mappings + * are stale and must be cleared along with compound state. + */ +static void fsdev_clear_folio_state(struct dev_dax *dev_dax) +{ + for (int i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + unsigned long pfn = PHYS_PFN(range->start); + unsigned long end_pfn = PHYS_PFN(range->end) + 1; + + while (pfn < end_pfn) { + struct folio *folio = pfn_folio(pfn); + int order = dax_folio_reset_order(folio); + + pfn += 1UL << order; + } + } +} + +static void fsdev_clear_folio_state_action(void *data) +{ + fsdev_clear_folio_state(data); +} + +static int fsdev_open(struct inode *inode, struct file *filp) +{ + struct dax_device *dax_dev = inode_dax(inode); + struct dev_dax *dev_dax = dax_get_private(dax_dev); + + filp->private_data = dev_dax; + + return 0; +} + +static int fsdev_release(struct inode *inode, struct file *filp) +{ + return 0; +} + +static const struct file_operations fsdev_fops = { + .llseek = noop_llseek, + .owner = THIS_MODULE, + .open = fsdev_open, + .release = fsdev_release, +}; + +static int fsdev_dax_probe(struct dev_dax *dev_dax) +{ + struct dax_device *dax_dev = dev_dax->dax_dev; + struct device *dev = &dev_dax->dev; + struct dev_pagemap *pgmap; + struct inode *inode; + u64 data_offset = 0; + struct cdev *cdev; + void *addr; + int rc, i; + + if (static_dev_dax(dev_dax)) { + if (dev_dax->nr_range > 1) { + dev_warn(dev, "static pgmap / multi-range device conflict\n"); + return -EINVAL; + } + + pgmap = dev_dax->pgmap; + } else { + size_t pgmap_size; + + if (dev_dax->pgmap) { + dev_warn(dev, "dynamic-dax with pre-populated page map\n"); + return -EINVAL; + } + + pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); + pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); + if (!pgmap) + return -ENOMEM; + + pgmap->nr_range = dev_dax->nr_range; + dev_dax->pgmap = pgmap; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + + pgmap->ranges[i] = *range; + } + } + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + + if (!devm_request_mem_region(dev, range->start, + range_len(range), dev_name(dev))) { + dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", + i, range->start, range->end); + return -EBUSY; + } + } + + /* Cache size now; it cannot change while driver is bound */ + dev_dax->cached_size = 0; + for (i = 0; i < dev_dax->nr_range; i++) + dev_dax->cached_size += range_len(&dev_dax->ranges[i].range); + + /* + * Use MEMORY_DEVICE_FS_DAX without setting vmemmap_shift, leaving + * folios at order-0. Unlike device.c (MEMORY_DEVICE_GENERIC), this + * lets fs-dax dynamically build compound folios as needed, similar + * to pmem behavior. + */ + pgmap->type = MEMORY_DEVICE_FS_DAX; + pgmap->ops = &fsdev_pagemap_ops; + pgmap->owner = dev_dax; + + addr = devm_memremap_pages(dev, pgmap); + if (IS_ERR(addr)) + return PTR_ERR(addr); + + /* + * Clear any stale compound folio state left over from a previous + * driver (e.g., device_dax with vmemmap_shift). Also register this + * as a devm action so folio state is cleared on unbind, ensuring + * clean pages for subsequent drivers (e.g., kmem for system-ram). + */ + fsdev_clear_folio_state(dev_dax); + rc = devm_add_action_or_reset(dev, fsdev_clear_folio_state_action, + dev_dax); + if (rc) + return rc; + + /* Detect whether the data is at a non-zero offset into the memory */ + if (pgmap->range.start != dev_dax->ranges[0].range.start) { + u64 phys = dev_dax->ranges[0].range.start; + u64 pgmap_phys = dev_dax->pgmap[0].range.start; + + if (!WARN_ON(pgmap_phys > phys)) + data_offset = phys - pgmap_phys; + + pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n", + __func__, phys, pgmap_phys, data_offset); + } + dev_dax->virt_addr = addr + data_offset; + + inode = dax_inode(dax_dev); + cdev = inode->i_cdev; + cdev_init(cdev, &fsdev_fops); + cdev->owner = dev->driver->owner; + cdev_set_parent(cdev, &dev->kobj); + rc = cdev_add(cdev, dev->devt, 1); + if (rc) + return rc; + + rc = devm_add_action_or_reset(dev, fsdev_cdev_del, cdev); + if (rc) + return rc; + + /* Set the dax operations for fs-dax access path */ + rc = dax_set_ops(dax_dev, &dev_dax_ops); + if (rc) + return rc; + + rc = devm_add_action_or_reset(dev, fsdev_clear_ops, dev_dax); + if (rc) + return rc; + + run_dax(dax_dev); + return devm_add_action_or_reset(dev, fsdev_kill, dev_dax); +} + +static struct dax_device_driver fsdev_dax_driver = { + .probe = fsdev_dax_probe, + .type = DAXDRV_FSDEV_TYPE, +}; + +static int __init dax_init(void) +{ + return dax_driver_register(&fsdev_dax_driver); +} + +static void __exit dax_exit(void) +{ + dax_driver_unregister(&fsdev_dax_driver); +} + +MODULE_AUTHOR("John Groves"); +MODULE_DESCRIPTION("FS-DAX Device: fs-dax compatible devdax driver"); +MODULE_LICENSE("GPL"); +module_init(dax_init); +module_exit(dax_exit); +MODULE_ALIAS_DAX_DEVICE(0); diff --git a/drivers/dax/hmem/Makefile b/drivers/dax/hmem/Makefile new file mode 100644 index 000000000..d4c4cd6bc --- /dev/null +++ b/drivers/dax/hmem/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# device_hmem.o deliberately precedes dax_hmem.o for initcall ordering +obj-$(CONFIG_DEV_DAX_HMEM_DEVICES) += device_hmem.o +obj-$(CONFIG_DEV_DAX_HMEM) += dax_hmem.o + +device_hmem-y := device.o +dax_hmem-y := hmem.o diff --git a/drivers/dax/hmem/device.c b/drivers/dax/hmem/device.c new file mode 100644 index 000000000..d70359b43 --- /dev/null +++ b/drivers/dax/hmem/device.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/platform_device.h> +#include <linux/memregion.h> +#include <linux/module.h> +#include <linux/dax.h> +#include <linux/mm.h> +#include "../bus.h" + +static bool nohmem; +module_param_named(disable, nohmem, bool, 0444); + +static bool platform_initialized; +static DEFINE_MUTEX(hmem_resource_lock); +static struct resource hmem_active = { + .name = "HMEM devices", + .start = 0, + .end = -1, + .flags = IORESOURCE_MEM, +}; + +int walk_hmem_resources(struct device *host, walk_hmem_fn fn) +{ + struct resource *res; + int rc = 0; + + mutex_lock(&hmem_resource_lock); + for (res = hmem_active.child; res; res = res->sibling) { + rc = fn(host, (int) res->desc, res); + if (rc) + break; + } + mutex_unlock(&hmem_resource_lock); + return rc; +} +EXPORT_SYMBOL_GPL(walk_hmem_resources); + +static void hmem_work(struct work_struct *work) +{ + /* place holder until dax_hmem driver attaches */ +} + +static struct hmem_platform_device hmem_platform = { + .pdev = { + .name = "hmem_platform", + .id = 0, + }, + .work = __WORK_INITIALIZER(hmem_platform.work, hmem_work), +}; + +static void __hmem_register_resource(int target_nid, struct resource *res) +{ + struct resource *new; + int rc; + + new = __request_region(&hmem_active, res->start, resource_size(res), "", + 0); + if (!new) { + pr_debug("hmem range %pr already active\n", res); + return; + } + + new->desc = target_nid; + + if (platform_initialized) + return; + + rc = platform_device_register(&hmem_platform.pdev); + if (rc) { + pr_err_once("failed to register device-dax hmem_platform device\n"); + return; + } + + platform_initialized = true; +} + +void hmem_register_resource(int target_nid, struct resource *res) +{ + if (nohmem) + return; + + mutex_lock(&hmem_resource_lock); + __hmem_register_resource(target_nid, res); + mutex_unlock(&hmem_resource_lock); +} + +static __init int hmem_register_one(struct resource *res, void *data) +{ + hmem_register_resource(phys_to_target_node(res->start), res); + + return 0; +} + +static __init int hmem_init(void) +{ + walk_soft_reserve_res(0, -1, NULL, hmem_register_one); + return 0; +} + +/* + * As this is a fallback for address ranges unclaimed by the ACPI HMAT + * parsing it must be at an initcall level greater than hmat_init(). + */ +device_initcall(hmem_init); diff --git a/drivers/dax/hmem/hmem.c b/drivers/dax/hmem/hmem.c new file mode 100644 index 000000000..af21f66bf --- /dev/null +++ b/drivers/dax/hmem/hmem.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/platform_device.h> +#include <linux/memregion.h> +#include <linux/module.h> +#include <linux/dax.h> +#include "../../cxl/cxl.h" +#include "../bus.h" + +static bool region_idle; +module_param_named(region_idle, region_idle, bool, 0644); + +static int dax_hmem_probe(struct platform_device *pdev) +{ + unsigned long flags = IORESOURCE_DAX_KMEM; + struct device *dev = &pdev->dev; + struct dax_region *dax_region; + struct memregion_info *mri; + struct dev_dax_data data; + + /* + * @region_idle == true indicates that an administrative agent + * wants to manipulate the range partitioning before the devices + * are created, so do not send them to the dax_kmem driver by + * default. + */ + if (region_idle) + flags = 0; + + mri = dev->platform_data; + dax_region = alloc_dax_region(dev, pdev->id, &mri->range, + mri->target_node, PMD_SIZE, flags); + if (!dax_region) + return -ENOMEM; + + data = (struct dev_dax_data) { + .dax_region = dax_region, + .id = -1, + .size = region_idle ? 0 : range_len(&mri->range), + .memmap_on_memory = false, + }; + + return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data)); +} + +static struct platform_driver dax_hmem_driver = { + .probe = dax_hmem_probe, + .driver = { + .name = "hmem", + }, +}; + +static void release_memregion(void *data) +{ + memregion_free((long) data); +} + +static void release_hmem(void *pdev) +{ + platform_device_unregister(pdev); +} + +static struct workqueue_struct *dax_hmem_wq; + +void dax_hmem_flush_work(void) +{ + flush_workqueue(dax_hmem_wq); +} +EXPORT_SYMBOL_FOR_MODULES(dax_hmem_flush_work, "dax_cxl"); + +static int __hmem_register_device(struct device *host, int target_nid, + const struct resource *res) +{ + struct platform_device *pdev; + struct memregion_info info; + long id; + int rc; + + rc = region_intersects_soft_reserve(res->start, resource_size(res)); + if (rc != REGION_INTERSECTS) + return 0; + + /* TODO: Add Soft-Reserved memory back to iomem */ + + id = memregion_alloc(GFP_KERNEL); + if (id < 0) { + dev_err(host, "memregion allocation failure for %pr\n", res); + return -ENOMEM; + } + rc = devm_add_action_or_reset(host, release_memregion, (void *) id); + if (rc) + return rc; + + pdev = platform_device_alloc("hmem", id); + if (!pdev) { + dev_err(host, "device allocation failure for %pr\n", res); + return -ENOMEM; + } + + pdev->dev.parent = host; + pdev->dev.numa_node = numa_map_to_online_node(target_nid); + info = (struct memregion_info) { + .target_node = target_nid, + .range = { + .start = res->start, + .end = res->end, + }, + }; + rc = platform_device_add_data(pdev, &info, sizeof(info)); + if (rc < 0) { + dev_err(host, "memregion_info allocation failure for %pr\n", + res); + goto out_put; + } + + rc = platform_device_add(pdev); + if (rc < 0) { + dev_err(host, "%s add failed for %pr\n", dev_name(&pdev->dev), + res); + goto out_put; + } + + return devm_add_action_or_reset(host, release_hmem, pdev); + +out_put: + platform_device_put(pdev); + return rc; +} + +static int hmem_register_cxl_device(struct device *host, int target_nid, + const struct resource *res) +{ + if (region_intersects(res->start, resource_size(res), IORESOURCE_MEM, + IORES_DESC_CXL) == REGION_DISJOINT) + return 0; + + if (cxl_region_contains_resource(res)) { + dev_dbg(host, "CXL claims resource, dropping: %pr\n", res); + return 0; + } + + dev_dbg(host, "CXL did not claim resource, registering: %pr\n", res); + return __hmem_register_device(host, target_nid, res); +} + +static void process_defer_work(struct work_struct *w) +{ + struct hmem_platform_device *hpdev = container_of(w, typeof(*hpdev), work); + struct device *dev = &hpdev->pdev.dev; + + /* Relies on cxl_acpi and cxl_pci having had a chance to load */ + wait_for_device_probe(); + + guard(device)(dev); + if (!dev->driver) + goto out; + + if (!hpdev->did_probe) { + hpdev->did_probe = true; + walk_hmem_resources(dev, hmem_register_cxl_device); + } +out: + put_device(dev); +} + +static int hmem_register_device(struct device *host, int target_nid, + const struct resource *res) +{ + struct platform_device *pdev = to_platform_device(host); + struct hmem_platform_device *hpdev = to_hmem_platform_device(pdev); + + if (IS_ENABLED(CONFIG_DEV_DAX_CXL) && + region_intersects(res->start, resource_size(res), IORESOURCE_MEM, + IORES_DESC_CXL) != REGION_DISJOINT) { + if (!hpdev->did_probe) { + dev_dbg(host, "await CXL initial probe: %pr\n", res); + hpdev->work.func = process_defer_work; + get_device(host); + if (!queue_work(dax_hmem_wq, &hpdev->work)) + put_device(host); + return 0; + } + dev_dbg(host, "deferring range to CXL: %pr\n", res); + return 0; + } + + return __hmem_register_device(host, target_nid, res); +} + +static int dax_hmem_platform_probe(struct platform_device *pdev) +{ + struct hmem_platform_device *hpdev = to_hmem_platform_device(pdev); + + /* queue is only flushed on module unload, fail rebind with pending work */ + if (work_pending(&hpdev->work)) + return -EBUSY; + + return walk_hmem_resources(&pdev->dev, hmem_register_device); +} + +static struct platform_driver dax_hmem_platform_driver = { + .probe = dax_hmem_platform_probe, + .driver = { + .name = "hmem_platform", + }, +}; + +static __init int dax_hmem_init(void) +{ + int rc; + + /* + * Ensure that cxl_acpi and cxl_pci have a chance to kick off + * CXL topology discovery at least once before scanning the + * iomem resource tree for IORES_DESC_CXL resources. + */ + if (IS_ENABLED(CONFIG_DEV_DAX_CXL)) { + request_module("cxl_acpi"); + request_module("cxl_pci"); + } + + dax_hmem_wq = alloc_ordered_workqueue("dax_hmem_wq", 0); + if (!dax_hmem_wq) + return -ENOMEM; + + rc = platform_driver_register(&dax_hmem_platform_driver); + if (rc) + goto err_platform_driver; + + rc = platform_driver_register(&dax_hmem_driver); + if (rc) + goto err_driver; + + return 0; + +err_driver: + platform_driver_unregister(&dax_hmem_platform_driver); +err_platform_driver: + destroy_workqueue(dax_hmem_wq); + + return rc; +} + +static __exit void dax_hmem_exit(void) +{ + platform_driver_unregister(&dax_hmem_driver); + platform_driver_unregister(&dax_hmem_platform_driver); + destroy_workqueue(dax_hmem_wq); +} + +module_init(dax_hmem_init); +module_exit(dax_hmem_exit); + +MODULE_ALIAS("platform:hmem*"); +MODULE_ALIAS("platform:hmem_platform*"); +MODULE_DESCRIPTION("HMEM DAX: direct access to 'specific purpose' memory"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c new file mode 100644 index 000000000..a18e2b968 --- /dev/null +++ b/drivers/dax/kmem.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> +#include <linux/pagemap.h> +#include <linux/memory.h> +#include <linux/module.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/dax.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/memory-tiers.h> +#include <linux/memory_hotplug.h> +#include <linux/string_helpers.h> +#include "dax-private.h" +#include "bus.h" + +/* + * Default abstract distance assigned to the NUMA node onlined + * by DAX/kmem if the low level platform driver didn't initialize + * one for this NUMA node. + */ +#define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5) + +/* Memory resource name used for add_memory_driver_managed(). */ +static const char *kmem_name; +/* Set if any memory will remain added when the driver will be unloaded. */ +static bool any_hotremove_failed; + +static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) +{ + struct dev_dax_range *dax_range = &dev_dax->ranges[i]; + struct range *range = &dax_range->range; + + /* memory-block align the hotplug range */ + r->start = ALIGN(range->start, memory_block_size_bytes()); + r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1; + if (r->start >= r->end) { + r->start = range->start; + r->end = range->end; + return -ENOSPC; + } + return 0; +} + +struct dax_kmem_data { + const char *res_name; + int mgid; + struct resource *res[]; +}; + +static DEFINE_MUTEX(kmem_memory_type_lock); +static LIST_HEAD(kmem_memory_types); + +static struct memory_dev_type *kmem_find_alloc_memory_type(int adist) +{ + guard(mutex)(&kmem_memory_type_lock); + return mt_find_alloc_memory_type(adist, &kmem_memory_types); +} + +static void kmem_put_memory_types(void) +{ + guard(mutex)(&kmem_memory_type_lock); + mt_put_memory_types(&kmem_memory_types); +} + +static int dev_dax_kmem_probe(struct dev_dax *dev_dax) +{ + struct device *dev = &dev_dax->dev; + unsigned long total_len = 0, orig_len = 0; + struct dax_kmem_data *data; + struct memory_dev_type *mtype; + int i, rc, mapped = 0; + mhp_t mhp_flags; + int numa_node; + int adist = MEMTIER_DEFAULT_DAX_ADISTANCE; + + /* + * Ensure good NUMA information for the persistent memory. + * Without this check, there is a risk that slow memory + * could be mixed in a node with faster memory, causing + * unavoidable performance issues. + */ + numa_node = dev_dax->target_node; + if (numa_node < 0) { + dev_warn(dev, "rejecting DAX region with invalid node: %d\n", + numa_node); + return -EINVAL; + } + + mt_calc_adistance(numa_node, &adist); + mtype = kmem_find_alloc_memory_type(adist); + if (IS_ERR(mtype)) + return PTR_ERR(mtype); + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range range; + + orig_len += range_len(&dev_dax->ranges[i].range); + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) { + dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n", + i, range.start, range.end); + continue; + } + total_len += range_len(&range); + } + + if (!total_len) { + dev_warn(dev, "rejecting DAX region without any memory after alignment\n"); + return -EINVAL; + } else if (total_len != orig_len) { + char buf[16]; + + string_get_size(orig_len - total_len, 1, STRING_UNITS_2, + buf, sizeof(buf)); + dev_warn(dev, "DAX region truncated by %s due to alignment\n", buf); + } + + init_node_memory_type(numa_node, mtype); + + rc = -ENOMEM; + data = kzalloc_flex(*data, res, dev_dax->nr_range); + if (!data) + goto err_dax_kmem_data; + + data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); + if (!data->res_name) + goto err_res_name; + + rc = memory_group_register_static(numa_node, PFN_UP(total_len)); + if (rc < 0) + goto err_reg_mgid; + data->mgid = rc; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct resource *res; + struct range range; + + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) + continue; + + /* Region is permanently reserved if hotremove fails. */ + res = request_mem_region(range.start, range_len(&range), data->res_name); + if (!res) { + dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", + i, range.start, range.end); + /* + * Once some memory has been onlined we can't + * assume that it can be un-onlined safely. + */ + if (mapped) + continue; + rc = -EBUSY; + goto err_request_mem; + } + data->res[i] = res; + + /* + * Set flags appropriate for System RAM. Leave ..._BUSY clear + * so that add_memory() can add a child resource. Do not + * inherit flags from the parent since it may set new flags + * unknown to us that will break add_memory() below. + */ + res->flags = IORESOURCE_SYSTEM_RAM; + + mhp_flags = MHP_NID_IS_MGID; + if (dev_dax->memmap_on_memory) + mhp_flags |= MHP_MEMMAP_ON_MEMORY; + + /* + * Ensure that future kexec'd kernels will not treat + * this as RAM automatically. + */ + rc = add_memory_driver_managed(data->mgid, range.start, + range_len(&range), kmem_name, mhp_flags); + + if (rc) { + dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", + i, range.start, range.end); + remove_resource(res); + kfree(res); + data->res[i] = NULL; + if (mapped) + continue; + goto err_request_mem; + } + mapped++; + } + + dev_set_drvdata(dev, data); + + return 0; + +err_request_mem: + memory_group_unregister(data->mgid); +err_reg_mgid: + kfree(data->res_name); +err_res_name: + kfree(data); +err_dax_kmem_data: + clear_node_memory_type(numa_node, mtype); + return rc; +} + +#ifdef CONFIG_MEMORY_HOTREMOVE +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) +{ + int i, success = 0; + int node = dev_dax->target_node; + struct device *dev = &dev_dax->dev; + struct dax_kmem_data *data = dev_get_drvdata(dev); + + /* + * We have one shot for removing memory, if some memory blocks were not + * offline prior to calling this function remove_memory() will fail, and + * there is no way to hotremove this memory until reboot because device + * unbind will succeed even if we return failure. + */ + for (i = 0; i < dev_dax->nr_range; i++) { + struct range range; + int rc; + + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) + continue; + + /* range was never added during probe */ + if (!data->res[i]) { + success++; + continue; + } + + rc = remove_memory(range.start, range_len(&range)); + if (rc == 0) { + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + success++; + continue; + } + any_hotremove_failed = true; + dev_err(dev, + "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n", + i, range.start, range.end); + } + + if (success >= dev_dax->nr_range) { + memory_group_unregister(data->mgid); + kfree(data->res_name); + kfree(data); + dev_set_drvdata(dev, NULL); + /* + * Clear the memtype association on successful unplug. + * If not, we have memory blocks left which can be + * offlined/onlined later. We need to keep memory_dev_type + * for that. This implies this reference will be around + * till next reboot. + */ + clear_node_memory_type(node, NULL); + } +} +#else +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) +{ + /* + * Without hotremove purposely leak the request_mem_region() for the + * device-dax range and return '0' to ->remove() attempts. The removal + * of the device from the driver always succeeds, but the region is + * permanently pinned as reserved by the unreleased + * request_mem_region(). + */ + any_hotremove_failed = true; +} +#endif /* CONFIG_MEMORY_HOTREMOVE */ + +static struct dax_device_driver device_dax_kmem_driver = { + .probe = dev_dax_kmem_probe, + .remove = dev_dax_kmem_remove, + .type = DAXDRV_KMEM_TYPE, +}; + +static int __init dax_kmem_init(void) +{ + int rc; + + /* Resource name is permanently allocated if any hotremove fails. */ + kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL); + if (!kmem_name) + return -ENOMEM; + + rc = dax_driver_register(&device_dax_kmem_driver); + if (rc) + goto error_dax_driver; + + return rc; + +error_dax_driver: + kmem_put_memory_types(); + kfree_const(kmem_name); + return rc; +} + +static void __exit dax_kmem_exit(void) +{ + dax_driver_unregister(&device_dax_kmem_driver); + if (!any_hotremove_failed) + kfree_const(kmem_name); + kmem_put_memory_types(); +} + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("KMEM DAX: map dax-devices as System-RAM"); +MODULE_LICENSE("GPL v2"); +module_init(dax_kmem_init); +module_exit(dax_kmem_exit); +MODULE_ALIAS_DAX_DEVICE(0); diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c new file mode 100644 index 000000000..bee93066a --- /dev/null +++ b/drivers/dax/pmem.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> +#include <linux/module.h> +#include "../nvdimm/pfn.h" +#include "../nvdimm/nd.h" +#include "bus.h" + +static struct dev_dax *__dax_pmem_probe(struct device *dev) +{ + struct range range; + int rc, id, region_id; + resource_size_t offset; + struct nd_pfn_sb *pfn_sb; + struct dev_dax_data data; + struct nd_namespace_io *nsio; + struct dax_region *dax_region; + struct dev_pagemap pgmap = { }; + struct nd_namespace_common *ndns; + struct nd_dax *nd_dax = to_nd_dax(dev); + struct nd_pfn *nd_pfn = &nd_dax->nd_pfn; + struct nd_region *nd_region = to_nd_region(dev->parent); + + ndns = nvdimm_namespace_common_probe(dev); + if (IS_ERR(ndns)) + return ERR_CAST(ndns); + + /* parse the 'pfn' info block via ->rw_bytes */ + rc = devm_namespace_enable(dev, ndns, nd_info_block_reserve()); + if (rc) + return ERR_PTR(rc); + rc = nvdimm_setup_pfn(nd_pfn, &pgmap); + if (rc) + return ERR_PTR(rc); + devm_namespace_disable(dev, ndns); + + /* reserve the metadata area, device-dax will reserve the data */ + pfn_sb = nd_pfn->pfn_sb; + offset = le64_to_cpu(pfn_sb->dataoff); + nsio = to_nd_namespace_io(&ndns->dev); + if (!devm_request_mem_region(dev, nsio->res.start, offset, + dev_name(&ndns->dev))) { + dev_warn(dev, "could not reserve metadata\n"); + return ERR_PTR(-EBUSY); + } + + rc = sscanf(dev_name(&ndns->dev), "namespace%d.%d", ®ion_id, &id); + if (rc != 2) + return ERR_PTR(-EINVAL); + + /* adjust the dax_region range to the start of data */ + range = pgmap.range; + range.start += offset; + dax_region = alloc_dax_region(dev, region_id, &range, + nd_region->target_node, le32_to_cpu(pfn_sb->align), + IORESOURCE_DAX_STATIC); + if (!dax_region) + return ERR_PTR(-ENOMEM); + + data = (struct dev_dax_data) { + .dax_region = dax_region, + .id = id, + .pgmap = &pgmap, + .size = range_len(&range), + .memmap_on_memory = false, + }; + + return devm_create_dev_dax(&data); +} + +static int dax_pmem_probe(struct device *dev) +{ + return PTR_ERR_OR_ZERO(__dax_pmem_probe(dev)); +} + +static struct nd_device_driver dax_pmem_driver = { + .probe = dax_pmem_probe, + .drv = { + .name = "dax_pmem", + }, + .type = ND_DRIVER_DAX_PMEM, +}; + +static int __init dax_pmem_init(void) +{ + return nd_driver_register(&dax_pmem_driver); +} +module_init(dax_pmem_init); + +static void __exit dax_pmem_exit(void) +{ + driver_unregister(&dax_pmem_driver.drv); +} +module_exit(dax_pmem_exit); + +MODULE_DESCRIPTION("PMEM DAX: direct access to persistent memory"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_ALIAS_ND_DEVICE(ND_DEVICE_DAX_PMEM); diff --git a/drivers/dax/super.c b/drivers/dax/super.c new file mode 100644 index 000000000..25cf99dd9 --- /dev/null +++ b/drivers/dax/super.c @@ -0,0 +1,712 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright(c) 2017 Intel Corporation. All rights reserved. + */ +#include <linux/pagemap.h> +#include <linux/module.h> +#include <linux/mount.h> +#include <linux/pseudo_fs.h> +#include <linux/magic.h> +#include <linux/cdev.h> +#include <linux/slab.h> +#include <linux/uio.h> +#include <linux/dax.h> +#include <linux/fs.h> +#include <linux/cacheinfo.h> +#include "dax-private.h" +#include "bus.h" + +/** + * struct dax_device - anchor object for dax services + * @inode: core vfs + * @cdev: optional character interface for "device dax" + * @private: dax driver private data + * @flags: state and boolean properties + * @ops: operations for this device + * @holder_data: holder of a dax_device: could be filesystem or mapped device + * @holder_ops: operations for the inner holder + */ +struct dax_device { + struct inode inode; + struct cdev cdev; + void *private; + unsigned long flags; + const struct dax_operations *ops; + void *holder_data; + const struct dax_holder_operations *holder_ops; +}; + +static dev_t dax_devt; +DEFINE_STATIC_SRCU(dax_srcu); +static struct vfsmount *dax_mnt; +static DEFINE_IDA(dax_minor_ida); +static struct kmem_cache *dax_cache __read_mostly; +static struct super_block *dax_superblock __read_mostly; + +int dax_read_lock(void) +{ + return srcu_read_lock(&dax_srcu); +} +EXPORT_SYMBOL_GPL(dax_read_lock); + +void dax_read_unlock(int id) +{ + srcu_read_unlock(&dax_srcu, id); +} +EXPORT_SYMBOL_GPL(dax_read_unlock); + +#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) +#include <linux/blkdev.h> + +static DEFINE_XARRAY(dax_hosts); + +int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) +{ + return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL); +} +EXPORT_SYMBOL_GPL(dax_add_host); + +void dax_remove_host(struct gendisk *disk) +{ + xa_erase(&dax_hosts, (unsigned long)disk); +} +EXPORT_SYMBOL_GPL(dax_remove_host); + +/** + * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax + * @bdev: block device to find a dax_device for + * @start_off: returns the byte offset into the dax_device that @bdev starts + * @holder: filesystem or mapped device inside the dax_device + * @ops: operations for the inner holder + */ +struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off, + void *holder, const struct dax_holder_operations *ops) +{ + struct dax_device *dax_dev; + u64 part_size; + int id; + + if (!blk_queue_dax(bdev->bd_disk->queue)) + return NULL; + + *start_off = get_start_sect(bdev) * SECTOR_SIZE; + part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE; + if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) { + pr_info("%pg: error: unaligned partition for dax\n", bdev); + return NULL; + } + + id = dax_read_lock(); + dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk); + if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode)) + dax_dev = NULL; + else if (holder) { + if (!cmpxchg(&dax_dev->holder_data, NULL, holder)) + dax_dev->holder_ops = ops; + else + dax_dev = NULL; + } + dax_read_unlock(id); + + return dax_dev; +} +EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); + +#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ + +#if IS_ENABLED(CONFIG_FS_DAX) + +void fs_put_dax(struct dax_device *dax_dev, void *holder) +{ + if (dax_dev && holder && + cmpxchg(&dax_dev->holder_data, holder, NULL) == holder) + dax_dev->holder_ops = NULL; + put_dax(dax_dev); +} +EXPORT_SYMBOL_GPL(fs_put_dax); + +/** + * fs_dax_get() - get ownership of a devdax via holder/holder_ops + * + * fs-dax file systems call this function to prepare to use a devdax device for + * fsdax. This is like fs_dax_get_by_bdev(), but the caller already has struct + * dev_dax (and there is no bdev). The holder makes this exclusive. + * + * @dax_dev: dev to be prepared for fs-dax usage + * @holder: filesystem or mapped device inside the dax_device + * @hops: operations for the inner holder + * + * Returns: 0 on success, <0 on failure + */ +int fs_dax_get(struct dax_device *dax_dev, void *holder, + const struct dax_holder_operations *hops) +{ + struct dev_dax *dev_dax; + struct dax_device_driver *dax_drv; + int id; + + id = dax_read_lock(); + if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode)) { + dax_read_unlock(id); + return -ENODEV; + } + dax_read_unlock(id); + + /* Verify the device is bound to fsdev_dax driver */ + dev_dax = dax_get_private(dax_dev); + if (!dev_dax) { + iput(&dax_dev->inode); + return -ENODEV; + } + + device_lock(&dev_dax->dev); + if (!dev_dax->dev.driver) { + device_unlock(&dev_dax->dev); + iput(&dax_dev->inode); + return -ENODEV; + } + dax_drv = to_dax_drv(dev_dax->dev.driver); + if (dax_drv->type != DAXDRV_FSDEV_TYPE) { + device_unlock(&dev_dax->dev); + iput(&dax_dev->inode); + return -EOPNOTSUPP; + } + device_unlock(&dev_dax->dev); + + if (cmpxchg(&dax_dev->holder_data, NULL, holder)) { + iput(&dax_dev->inode); + return -EBUSY; + } + + dax_dev->holder_ops = hops; + + return 0; +} +EXPORT_SYMBOL_GPL(fs_dax_get); +#endif /* CONFIG_FS_DAX */ + +enum dax_device_flags { + /* !alive + rcu grace period == no new operations / mappings */ + DAXDEV_ALIVE, + /* gate whether dax_flush() calls the low level flush routine */ + DAXDEV_WRITE_CACHE, + /* flag to check if device supports synchronous flush */ + DAXDEV_SYNC, + /* do not leave the caches dirty after writes */ + DAXDEV_NOCACHE, + /* handle CPU fetch exceptions during reads */ + DAXDEV_NOMC, +}; + +/** + * dax_direct_access() - translate a device pgoff to an absolute pfn + * @dax_dev: a dax_device instance representing the logical memory range + * @pgoff: offset in pages from the start of the device to translate + * @nr_pages: number of consecutive pages caller can handle relative to @pfn + * @mode: indicator on normal access or recovery write + * @kaddr: output parameter that returns a virtual address mapping of pfn + * @pfn: output parameter that returns an absolute pfn translation of @pgoff + * + * Return: negative errno if an error occurs, otherwise the number of + * pages accessible at the device relative @pgoff. + */ +long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, + enum dax_access_mode mode, void **kaddr, unsigned long *pfn) +{ + long avail; + + if (!dax_dev) + return -EOPNOTSUPP; + + if (!dax_alive(dax_dev)) + return -ENXIO; + + if (!dax_dev->ops) + return -EOPNOTSUPP; + + if (nr_pages < 0) + return -EINVAL; + + avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages, + mode, kaddr, pfn); + if (!avail) + return -ERANGE; + return min(avail, nr_pages); +} +EXPORT_SYMBOL_GPL(dax_direct_access); + +size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, + size_t bytes, struct iov_iter *i) +{ + if (!dax_alive(dax_dev)) + return 0; + + /* + * The userspace address for the memory copy has already been validated + * via access_ok() in vfs_write, so use the 'no check' version to bypass + * the HARDENED_USERCOPY overhead. + */ + if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags)) + return _copy_from_iter_flushcache(addr, bytes, i); + return _copy_from_iter(addr, bytes, i); +} + +size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, + size_t bytes, struct iov_iter *i) +{ + if (!dax_alive(dax_dev)) + return 0; + + /* + * The userspace address for the memory copy has already been validated + * via access_ok() in vfs_red, so use the 'no check' version to bypass + * the HARDENED_USERCOPY overhead. + */ + if (test_bit(DAXDEV_NOMC, &dax_dev->flags)) + return _copy_mc_to_iter(addr, bytes, i); + return _copy_to_iter(addr, bytes, i); +} + +int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages) +{ + int ret; + + if (!dax_alive(dax_dev)) + return -ENXIO; + + if (!dax_dev->ops) + return -EOPNOTSUPP; + + /* + * There are no callers that want to zero more than one page as of now. + * Once users are there, this check can be removed after the + * device mapper code has been updated to split ranges across targets. + */ + if (nr_pages != 1) + return -EIO; + + ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages); + return dax_mem2blk_err(ret); +} +EXPORT_SYMBOL_GPL(dax_zero_page_range); + +size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, + void *addr, size_t bytes, struct iov_iter *iter) +{ + if (!dax_dev->ops || !dax_dev->ops->recovery_write) + return 0; + return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter); +} +EXPORT_SYMBOL_GPL(dax_recovery_write); + +int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, + u64 len, int mf_flags) +{ + int rc, id; + + id = dax_read_lock(); + if (!dax_alive(dax_dev)) { + rc = -ENXIO; + goto out; + } + + if (!dax_dev->holder_ops) { + rc = -EOPNOTSUPP; + goto out; + } + + rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags); +out: + dax_read_unlock(id); + return rc; +} +EXPORT_SYMBOL_GPL(dax_holder_notify_failure); + +#ifdef CONFIG_ARCH_HAS_PMEM_API +void arch_wb_cache_pmem(void *addr, size_t size); +void dax_flush(struct dax_device *dax_dev, void *addr, size_t size) +{ + if (unlikely(!dax_write_cache_enabled(dax_dev))) + return; + + arch_wb_cache_pmem(addr, size); +} +#else +void dax_flush(struct dax_device *dax_dev, void *addr, size_t size) +{ +} +#endif +EXPORT_SYMBOL_GPL(dax_flush); + +void dax_write_cache(struct dax_device *dax_dev, bool wc) +{ + if (wc) + set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags); + else + clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(dax_write_cache); + +bool dax_write_cache_enabled(struct dax_device *dax_dev) +{ + return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(dax_write_cache_enabled); + +bool dax_synchronous(struct dax_device *dax_dev) +{ + return test_bit(DAXDEV_SYNC, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(dax_synchronous); + +void set_dax_synchronous(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_SYNC, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(set_dax_synchronous); + +void set_dax_nocache(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_NOCACHE, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(set_dax_nocache); + +void set_dax_nomc(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_NOMC, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(set_dax_nomc); + +/** + * dax_set_ops - set the dax_operations for a dax_device + * @dax_dev: the dax_device to configure + * @ops: the operations to set (may be NULL to clear) + * + * This allows drivers to set the dax_operations after the dax_device + * has been allocated. This is needed when the device is created before + * the driver that needs specific ops is bound (e.g., fsdev_dax binding + * to a dev_dax created by hmem). + * + * When setting non-NULL ops, fails if ops are already set (returns -EBUSY). + * When clearing ops (NULL), always succeeds. + * + * Return: 0 on success, -EBUSY if ops already set + */ +int dax_set_ops(struct dax_device *dax_dev, const struct dax_operations *ops) +{ + if (ops) { + /* Setting ops: fail if already set */ + if (cmpxchg(&dax_dev->ops, NULL, ops) != NULL) + return -EBUSY; + } else { + /* Clearing ops: always allowed */ + dax_dev->ops = NULL; + } + return 0; +} +EXPORT_SYMBOL_GPL(dax_set_ops); + +bool dax_alive(struct dax_device *dax_dev) +{ + lockdep_assert_held(&dax_srcu); + return test_bit(DAXDEV_ALIVE, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(dax_alive); + +/* + * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring + * that any fault handlers or operations that might have seen + * dax_alive(), have completed. Any operations that start after + * synchronize_srcu() has run will abort upon seeing !dax_alive(). + * + * Note, because alloc_dax() returns an ERR_PTR() on error, callers + * typically store its result into a local variable in order to check + * the result. Therefore, care must be taken to populate the struct + * device dax_dev field make sure the dax_dev is not leaked. + */ +void kill_dax(struct dax_device *dax_dev) +{ + if (!dax_dev) + return; + + if (dax_dev->holder_data != NULL) + dax_holder_notify_failure(dax_dev, 0, U64_MAX, + MF_MEM_PRE_REMOVE); + + clear_bit(DAXDEV_ALIVE, &dax_dev->flags); + synchronize_srcu(&dax_srcu); + + /* clear holder data */ + dax_dev->holder_ops = NULL; + dax_dev->holder_data = NULL; +} +EXPORT_SYMBOL_GPL(kill_dax); + +void run_dax(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_ALIVE, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(run_dax); + +static struct inode *dax_alloc_inode(struct super_block *sb) +{ + struct dax_device *dax_dev; + struct inode *inode; + + dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL); + if (!dax_dev) + return NULL; + + inode = &dax_dev->inode; + inode->i_rdev = 0; + return inode; +} + +static struct dax_device *to_dax_dev(struct inode *inode) +{ + return container_of(inode, struct dax_device, inode); +} + +static void dax_free_inode(struct inode *inode) +{ + struct dax_device *dax_dev = to_dax_dev(inode); + if (inode->i_rdev) + ida_free(&dax_minor_ida, iminor(inode)); + kmem_cache_free(dax_cache, dax_dev); +} + +static void dax_destroy_inode(struct inode *inode) +{ + struct dax_device *dax_dev = to_dax_dev(inode); + WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags), + "kill_dax() must be called before final iput()\n"); +} + +static const struct super_operations dax_sops = { + .statfs = simple_statfs, + .alloc_inode = dax_alloc_inode, + .destroy_inode = dax_destroy_inode, + .free_inode = dax_free_inode, + .drop_inode = inode_just_drop, +}; + +static int dax_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC); + if (!ctx) + return -ENOMEM; + ctx->ops = &dax_sops; + return 0; +} + +static struct file_system_type dax_fs_type = { + .name = "dax", + .init_fs_context = dax_init_fs_context, + .kill_sb = kill_anon_super, +}; + +static int dax_test(struct inode *inode, void *data) +{ + dev_t devt = *(dev_t *) data; + + return inode->i_rdev == devt; +} + +static int dax_set(struct inode *inode, void *data) +{ + dev_t devt = *(dev_t *) data; + + inode->i_rdev = devt; + return 0; +} + +struct dax_device *dax_dev_get(dev_t devt) +{ + struct dax_device *dax_dev; + struct inode *inode; + + inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31), + dax_test, dax_set, &devt); + + if (!inode) + return NULL; + + dax_dev = to_dax_dev(inode); + if (inode_state_read_once(inode) & I_NEW) { + set_bit(DAXDEV_ALIVE, &dax_dev->flags); + inode->i_cdev = &dax_dev->cdev; + inode->i_mode = S_IFCHR; + inode->i_flags = S_DAX; + mapping_set_gfp_mask(&inode->i_data, GFP_USER); + unlock_new_inode(inode); + } + + return dax_dev; +} +EXPORT_SYMBOL_GPL(dax_dev_get); + +struct dax_device *alloc_dax(void *private, const struct dax_operations *ops) +{ + struct dax_device *dax_dev; + dev_t devt; + int minor; + + /* + * Unavailable on architectures with virtually aliased data caches, + * except for device-dax (NULL operations pointer), which does + * not use aliased mappings from the kernel. + */ + if (ops && cpu_dcache_is_aliasing()) + return ERR_PTR(-EOPNOTSUPP); + + if (WARN_ON_ONCE(ops && !ops->zero_page_range)) + return ERR_PTR(-EINVAL); + + minor = ida_alloc_max(&dax_minor_ida, MINORMASK, GFP_KERNEL); + if (minor < 0) + return ERR_PTR(-ENOMEM); + + devt = MKDEV(MAJOR(dax_devt), minor); + dax_dev = dax_dev_get(devt); + if (!dax_dev) + goto err_dev; + + dax_dev->ops = ops; + dax_dev->private = private; + return dax_dev; + + err_dev: + ida_free(&dax_minor_ida, minor); + return ERR_PTR(-ENOMEM); +} +EXPORT_SYMBOL_GPL(alloc_dax); + +void put_dax(struct dax_device *dax_dev) +{ + if (!dax_dev) + return; + iput(&dax_dev->inode); +} +EXPORT_SYMBOL_GPL(put_dax); + +/** + * dax_holder() - obtain the holder of a dax device + * @dax_dev: a dax_device instance + * + * Return: the holder's data which represents the holder if registered, + * otherwize NULL. + */ +void *dax_holder(struct dax_device *dax_dev) +{ + return dax_dev->holder_data; +} +EXPORT_SYMBOL_GPL(dax_holder); + +/** + * inode_dax: convert a public inode into its dax_dev + * @inode: An inode with i_cdev pointing to a dax_dev + * + * Note this is not equivalent to to_dax_dev() which is for private + * internal use where we know the inode filesystem type == dax_fs_type. + */ +struct dax_device *inode_dax(struct inode *inode) +{ + struct cdev *cdev = inode->i_cdev; + + return container_of(cdev, struct dax_device, cdev); +} +EXPORT_SYMBOL_GPL(inode_dax); + +struct inode *dax_inode(struct dax_device *dax_dev) +{ + return &dax_dev->inode; +} +EXPORT_SYMBOL_GPL(dax_inode); + +void *dax_get_private(struct dax_device *dax_dev) +{ + if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags)) + return NULL; + return dax_dev->private; +} +EXPORT_SYMBOL_GPL(dax_get_private); + +static void init_once(void *_dax_dev) +{ + struct dax_device *dax_dev = _dax_dev; + struct inode *inode = &dax_dev->inode; + + memset(dax_dev, 0, sizeof(*dax_dev)); + inode_init_once(inode); +} + +static int dax_fs_init(void) +{ + int rc; + + dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0, + SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + init_once); + if (!dax_cache) + return -ENOMEM; + + dax_mnt = kern_mount(&dax_fs_type); + if (IS_ERR(dax_mnt)) { + rc = PTR_ERR(dax_mnt); + goto err_mount; + } + dax_superblock = dax_mnt->mnt_sb; + + return 0; + + err_mount: + kmem_cache_destroy(dax_cache); + + return rc; +} + +static void dax_fs_exit(void) +{ + kern_unmount(dax_mnt); + rcu_barrier(); + kmem_cache_destroy(dax_cache); +} + +static int __init dax_core_init(void) +{ + int rc; + + rc = dax_fs_init(); + if (rc) + return rc; + + rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax"); + if (rc) + goto err_chrdev; + + rc = dax_bus_init(); + if (rc) + goto err_bus; + return 0; + +err_bus: + unregister_chrdev_region(dax_devt, MINORMASK+1); +err_chrdev: + dax_fs_exit(); + return 0; +} + +static void __exit dax_core_exit(void) +{ + dax_bus_exit(); + unregister_chrdev_region(dax_devt, MINORMASK+1); + ida_destroy(&dax_minor_ida); + dax_fs_exit(); +} + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("DAX: direct access to differentiated memory"); +MODULE_LICENSE("GPL v2"); +subsys_initcall(dax_core_init); +module_exit(dax_core_exit); |