diff options
Diffstat (limited to 'drivers/dax/hmem')
| -rw-r--r-- | drivers/dax/hmem/Makefile | 7 | ||||
| -rw-r--r-- | drivers/dax/hmem/device.c | 103 | ||||
| -rw-r--r-- | drivers/dax/hmem/hmem.c | 257 |
3 files changed, 367 insertions, 0 deletions
diff --git a/drivers/dax/hmem/Makefile b/drivers/dax/hmem/Makefile new file mode 100644 index 000000000..d4c4cd6bc --- /dev/null +++ b/drivers/dax/hmem/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# device_hmem.o deliberately precedes dax_hmem.o for initcall ordering +obj-$(CONFIG_DEV_DAX_HMEM_DEVICES) += device_hmem.o +obj-$(CONFIG_DEV_DAX_HMEM) += dax_hmem.o + +device_hmem-y := device.o +dax_hmem-y := hmem.o diff --git a/drivers/dax/hmem/device.c b/drivers/dax/hmem/device.c new file mode 100644 index 000000000..d70359b43 --- /dev/null +++ b/drivers/dax/hmem/device.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/platform_device.h> +#include <linux/memregion.h> +#include <linux/module.h> +#include <linux/dax.h> +#include <linux/mm.h> +#include "../bus.h" + +static bool nohmem; +module_param_named(disable, nohmem, bool, 0444); + +static bool platform_initialized; +static DEFINE_MUTEX(hmem_resource_lock); +static struct resource hmem_active = { + .name = "HMEM devices", + .start = 0, + .end = -1, + .flags = IORESOURCE_MEM, +}; + +int walk_hmem_resources(struct device *host, walk_hmem_fn fn) +{ + struct resource *res; + int rc = 0; + + mutex_lock(&hmem_resource_lock); + for (res = hmem_active.child; res; res = res->sibling) { + rc = fn(host, (int) res->desc, res); + if (rc) + break; + } + mutex_unlock(&hmem_resource_lock); + return rc; +} +EXPORT_SYMBOL_GPL(walk_hmem_resources); + +static void hmem_work(struct work_struct *work) +{ + /* place holder until dax_hmem driver attaches */ +} + +static struct hmem_platform_device hmem_platform = { + .pdev = { + .name = "hmem_platform", + .id = 0, + }, + .work = __WORK_INITIALIZER(hmem_platform.work, hmem_work), +}; + +static void __hmem_register_resource(int target_nid, struct resource *res) +{ + struct resource *new; + int rc; + + new = __request_region(&hmem_active, res->start, resource_size(res), "", + 0); + if (!new) { + pr_debug("hmem range %pr already active\n", res); + return; + } + + new->desc = target_nid; + + if (platform_initialized) + return; + + rc = platform_device_register(&hmem_platform.pdev); + if (rc) { + pr_err_once("failed to register device-dax hmem_platform device\n"); + return; + } + + platform_initialized = true; +} + +void hmem_register_resource(int target_nid, struct resource *res) +{ + if (nohmem) + return; + + mutex_lock(&hmem_resource_lock); + __hmem_register_resource(target_nid, res); + mutex_unlock(&hmem_resource_lock); +} + +static __init int hmem_register_one(struct resource *res, void *data) +{ + hmem_register_resource(phys_to_target_node(res->start), res); + + return 0; +} + +static __init int hmem_init(void) +{ + walk_soft_reserve_res(0, -1, NULL, hmem_register_one); + return 0; +} + +/* + * As this is a fallback for address ranges unclaimed by the ACPI HMAT + * parsing it must be at an initcall level greater than hmat_init(). + */ +device_initcall(hmem_init); diff --git a/drivers/dax/hmem/hmem.c b/drivers/dax/hmem/hmem.c new file mode 100644 index 000000000..af21f66bf --- /dev/null +++ b/drivers/dax/hmem/hmem.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/platform_device.h> +#include <linux/memregion.h> +#include <linux/module.h> +#include <linux/dax.h> +#include "../../cxl/cxl.h" +#include "../bus.h" + +static bool region_idle; +module_param_named(region_idle, region_idle, bool, 0644); + +static int dax_hmem_probe(struct platform_device *pdev) +{ + unsigned long flags = IORESOURCE_DAX_KMEM; + struct device *dev = &pdev->dev; + struct dax_region *dax_region; + struct memregion_info *mri; + struct dev_dax_data data; + + /* + * @region_idle == true indicates that an administrative agent + * wants to manipulate the range partitioning before the devices + * are created, so do not send them to the dax_kmem driver by + * default. + */ + if (region_idle) + flags = 0; + + mri = dev->platform_data; + dax_region = alloc_dax_region(dev, pdev->id, &mri->range, + mri->target_node, PMD_SIZE, flags); + if (!dax_region) + return -ENOMEM; + + data = (struct dev_dax_data) { + .dax_region = dax_region, + .id = -1, + .size = region_idle ? 0 : range_len(&mri->range), + .memmap_on_memory = false, + }; + + return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data)); +} + +static struct platform_driver dax_hmem_driver = { + .probe = dax_hmem_probe, + .driver = { + .name = "hmem", + }, +}; + +static void release_memregion(void *data) +{ + memregion_free((long) data); +} + +static void release_hmem(void *pdev) +{ + platform_device_unregister(pdev); +} + +static struct workqueue_struct *dax_hmem_wq; + +void dax_hmem_flush_work(void) +{ + flush_workqueue(dax_hmem_wq); +} +EXPORT_SYMBOL_FOR_MODULES(dax_hmem_flush_work, "dax_cxl"); + +static int __hmem_register_device(struct device *host, int target_nid, + const struct resource *res) +{ + struct platform_device *pdev; + struct memregion_info info; + long id; + int rc; + + rc = region_intersects_soft_reserve(res->start, resource_size(res)); + if (rc != REGION_INTERSECTS) + return 0; + + /* TODO: Add Soft-Reserved memory back to iomem */ + + id = memregion_alloc(GFP_KERNEL); + if (id < 0) { + dev_err(host, "memregion allocation failure for %pr\n", res); + return -ENOMEM; + } + rc = devm_add_action_or_reset(host, release_memregion, (void *) id); + if (rc) + return rc; + + pdev = platform_device_alloc("hmem", id); + if (!pdev) { + dev_err(host, "device allocation failure for %pr\n", res); + return -ENOMEM; + } + + pdev->dev.parent = host; + pdev->dev.numa_node = numa_map_to_online_node(target_nid); + info = (struct memregion_info) { + .target_node = target_nid, + .range = { + .start = res->start, + .end = res->end, + }, + }; + rc = platform_device_add_data(pdev, &info, sizeof(info)); + if (rc < 0) { + dev_err(host, "memregion_info allocation failure for %pr\n", + res); + goto out_put; + } + + rc = platform_device_add(pdev); + if (rc < 0) { + dev_err(host, "%s add failed for %pr\n", dev_name(&pdev->dev), + res); + goto out_put; + } + + return devm_add_action_or_reset(host, release_hmem, pdev); + +out_put: + platform_device_put(pdev); + return rc; +} + +static int hmem_register_cxl_device(struct device *host, int target_nid, + const struct resource *res) +{ + if (region_intersects(res->start, resource_size(res), IORESOURCE_MEM, + IORES_DESC_CXL) == REGION_DISJOINT) + return 0; + + if (cxl_region_contains_resource(res)) { + dev_dbg(host, "CXL claims resource, dropping: %pr\n", res); + return 0; + } + + dev_dbg(host, "CXL did not claim resource, registering: %pr\n", res); + return __hmem_register_device(host, target_nid, res); +} + +static void process_defer_work(struct work_struct *w) +{ + struct hmem_platform_device *hpdev = container_of(w, typeof(*hpdev), work); + struct device *dev = &hpdev->pdev.dev; + + /* Relies on cxl_acpi and cxl_pci having had a chance to load */ + wait_for_device_probe(); + + guard(device)(dev); + if (!dev->driver) + goto out; + + if (!hpdev->did_probe) { + hpdev->did_probe = true; + walk_hmem_resources(dev, hmem_register_cxl_device); + } +out: + put_device(dev); +} + +static int hmem_register_device(struct device *host, int target_nid, + const struct resource *res) +{ + struct platform_device *pdev = to_platform_device(host); + struct hmem_platform_device *hpdev = to_hmem_platform_device(pdev); + + if (IS_ENABLED(CONFIG_DEV_DAX_CXL) && + region_intersects(res->start, resource_size(res), IORESOURCE_MEM, + IORES_DESC_CXL) != REGION_DISJOINT) { + if (!hpdev->did_probe) { + dev_dbg(host, "await CXL initial probe: %pr\n", res); + hpdev->work.func = process_defer_work; + get_device(host); + if (!queue_work(dax_hmem_wq, &hpdev->work)) + put_device(host); + return 0; + } + dev_dbg(host, "deferring range to CXL: %pr\n", res); + return 0; + } + + return __hmem_register_device(host, target_nid, res); +} + +static int dax_hmem_platform_probe(struct platform_device *pdev) +{ + struct hmem_platform_device *hpdev = to_hmem_platform_device(pdev); + + /* queue is only flushed on module unload, fail rebind with pending work */ + if (work_pending(&hpdev->work)) + return -EBUSY; + + return walk_hmem_resources(&pdev->dev, hmem_register_device); +} + +static struct platform_driver dax_hmem_platform_driver = { + .probe = dax_hmem_platform_probe, + .driver = { + .name = "hmem_platform", + }, +}; + +static __init int dax_hmem_init(void) +{ + int rc; + + /* + * Ensure that cxl_acpi and cxl_pci have a chance to kick off + * CXL topology discovery at least once before scanning the + * iomem resource tree for IORES_DESC_CXL resources. + */ + if (IS_ENABLED(CONFIG_DEV_DAX_CXL)) { + request_module("cxl_acpi"); + request_module("cxl_pci"); + } + + dax_hmem_wq = alloc_ordered_workqueue("dax_hmem_wq", 0); + if (!dax_hmem_wq) + return -ENOMEM; + + rc = platform_driver_register(&dax_hmem_platform_driver); + if (rc) + goto err_platform_driver; + + rc = platform_driver_register(&dax_hmem_driver); + if (rc) + goto err_driver; + + return 0; + +err_driver: + platform_driver_unregister(&dax_hmem_platform_driver); +err_platform_driver: + destroy_workqueue(dax_hmem_wq); + + return rc; +} + +static __exit void dax_hmem_exit(void) +{ + platform_driver_unregister(&dax_hmem_driver); + platform_driver_unregister(&dax_hmem_platform_driver); + destroy_workqueue(dax_hmem_wq); +} + +module_init(dax_hmem_init); +module_exit(dax_hmem_exit); + +MODULE_ALIAS("platform:hmem*"); +MODULE_ALIAS("platform:hmem_platform*"); +MODULE_DESCRIPTION("HMEM DAX: direct access to 'specific purpose' memory"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); |