diff options
| author | Sang-Heon Jeon <ekffu200098@gmail.com> | 2026-08-24 01:18:14 +0900 |
|---|---|---|
| committer | Julia Lawall <Julia.Lawall@inria.fr> | 2026-08-23 22:12:11 +0200 |
| commit | 2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c (patch) | |
| tree | dcc4da1d99f17541c5590ac63c77da4bf372a4ab /mm/sparse.c | |
| download | linux-stable-2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c.tar.gz linux-stable-2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c.zip | |
coccinelle: alloc_cast: drop removed allocatorsgrafted
- pci_alloc_consistent() and pci_zalloc_consistent() were removed by
commit 7968778914e5
("PCI: Remove the deprecated "pci-dma-compat.h" API")
- kmem_alloc() was removed by commit f078d4ea8276
("xfs: convert kmem_alloc() to kmalloc()")
- kmem_zalloc() was removed by commit 10634530f7ba
("xfs: convert kmem_zalloc() to kzalloc()")
- kmem_zone_alloc() and kmem_zone_zalloc() were removed by commit
bae633a4a283 ("xfs: remove xfs_zone_{alloc,zalloc} helpers")
So drop them from the rules.
No functional change.
Signed-off-by: Sang-Heon Jeon <ekffu200098@gmail.com>
Signed-off-by: Julia Lawall <Julia.Lawall@inria.fr>
Diffstat (limited to 'mm/sparse.c')
| -rw-r--r-- | mm/sparse.c | 364 |
1 files changed, 364 insertions, 0 deletions
diff --git a/mm/sparse.c b/mm/sparse.c new file mode 100644 index 000000000..704a9dec2 --- /dev/null +++ b/mm/sparse.c @@ -0,0 +1,364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sparse memory mappings. + */ +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/mmzone.h> +#include <linux/memblock.h> +#include <linux/compiler.h> +#include <linux/highmem.h> +#include <linux/export.h> +#include <linux/spinlock.h> +#include <linux/vmalloc.h> +#include <linux/swap.h> +#include <linux/swapops.h> +#include <linux/vmstat.h> +#include "internal.h" +#include "mm_init.h" +#include "sparse.h" +#include <asm/dma.h> + +/* + * Permanent SPARSEMEM data: + * + * 1) mem_section - memory sections, mem_map's for valid memory + */ +#ifdef CONFIG_SPARSEMEM_EXTREME +struct mem_section **mem_section; +#else +struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] + ____cacheline_internodealigned_in_smp; +#endif +EXPORT_SYMBOL(mem_section); + +#ifdef NODE_NOT_IN_PAGE_FLAGS +/* + * If we did not store the node number in the page then we have to + * do a lookup in the section_to_node_table in order to find which + * node the page belongs to. + */ +#if MAX_NUMNODES <= 256 +static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; +#else +static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; +#endif + +int memdesc_nid(const memdesc_flags_t *mdf) +{ + return section_to_node_table[memdesc_section(mdf)]; +} +EXPORT_SYMBOL(memdesc_nid); + +static void set_section_nid(unsigned long section_nr, int nid) +{ + section_to_node_table[section_nr] = nid; +} +#else /* !NODE_NOT_IN_PAGE_FLAGS */ +static inline void set_section_nid(unsigned long section_nr, int nid) +{ +} +#endif + +#ifdef CONFIG_SPARSEMEM_EXTREME +static noinline struct mem_section __ref *sparse_index_alloc(int nid) +{ + struct mem_section *section = NULL; + unsigned long array_size = SECTIONS_PER_ROOT * + sizeof(struct mem_section); + + if (slab_is_available()) { + section = kzalloc_node(array_size, GFP_KERNEL, nid); + } else { + section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, + nid); + if (!section) + panic("%s: Failed to allocate %lu bytes nid=%d\n", + __func__, array_size, nid); + } + + return section; +} + +int __meminit sparse_index_init(unsigned long section_nr, int nid) +{ + unsigned long root = SECTION_NR_TO_ROOT(section_nr); + struct mem_section *section; + + /* + * An existing section is possible in the sub-section hotplug + * case. First hot-add instantiates, follow-on hot-add reuses + * the existing section. + * + * The mem_hotplug_lock resolves the apparent race below. + */ + if (mem_section[root]) + return 0; + + section = sparse_index_alloc(nid); + if (!section) + return -ENOMEM; + + mem_section[root] = section; + + return 0; +} +#else /* !SPARSEMEM_EXTREME */ +int sparse_index_init(unsigned long section_nr, int nid) +{ + return 0; +} +#endif + +/* + * During early boot, before section_mem_map is used for an actual + * mem_map, we use section_mem_map to store the section's NUMA + * node. This keeps us from having to use another data structure. The + * node information is cleared just before we store the real mem_map. + */ +static inline unsigned long sparse_encode_early_nid(int nid) +{ + return ((unsigned long)nid << SECTION_NID_SHIFT); +} + +static inline int sparse_early_nid(struct mem_section *section) +{ + return (section->section_mem_map >> SECTION_NID_SHIFT); +} + +/* Validate the physical addressing limitations of the model */ +static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, + unsigned long *end_pfn) +{ + unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT; + + /* + * Sanity checks - do not allow an architecture to pass + * in larger pfns than the maximum scope of sparsemem: + */ + if (*start_pfn > max_sparsemem_pfn) { + mminit_dprintk(MMINIT_WARNING, "pfnvalidation", + "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", + *start_pfn, *end_pfn, max_sparsemem_pfn); + WARN_ON_ONCE(1); + *start_pfn = max_sparsemem_pfn; + *end_pfn = max_sparsemem_pfn; + } else if (*end_pfn > max_sparsemem_pfn) { + mminit_dprintk(MMINIT_WARNING, "pfnvalidation", + "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", + *start_pfn, *end_pfn, max_sparsemem_pfn); + WARN_ON_ONCE(1); + *end_pfn = max_sparsemem_pfn; + } +} + +/* + * There are a number of times that we loop over NR_MEM_SECTIONS, + * looking for section_present() on each. But, when we have very + * large physical address spaces, NR_MEM_SECTIONS can also be + * very large which makes the loops quite long. + * + * Keeping track of this gives us an easy way to break out of + * those loops early. + */ +unsigned long __highest_present_section_nr; + +static inline unsigned long first_present_section_nr(void) +{ + return next_present_section_nr(-1); +} + +/* Record a memory area against a node. */ +static void __init memory_present(int nid, unsigned long start, unsigned long end) +{ + unsigned long pfn; + + start &= PAGE_SECTION_MASK; + mminit_validate_memmodel_limits(&start, &end); + for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { + unsigned long section_nr = pfn_to_section_nr(pfn); + struct mem_section *ms; + + sparse_index_init(section_nr, nid); + set_section_nid(section_nr, nid); + + ms = __nr_to_section(section_nr); + if (!ms->section_mem_map) { + ms->section_mem_map = sparse_encode_early_nid(nid) | + SECTION_IS_ONLINE; + __section_mark_present(ms, section_nr); + } + } +} + +/* + * Mark all memblocks as present using memory_present(). + * This is a convenience function that is useful to mark all of the systems + * memory as present during initialization. + */ +static void __init memblocks_present(void) +{ + unsigned long start, end; + int i, nid; + +#ifdef CONFIG_SPARSEMEM_EXTREME + unsigned long size, align; + + size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; + align = 1 << (INTERNODE_CACHE_SHIFT); + mem_section = memblock_alloc_or_panic(size, align); +#endif + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) + memory_present(nid, start, end); +} + +static unsigned long usemap_size(void) +{ + return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long); +} + +size_t mem_section_usage_size(void) +{ + return sizeof(struct mem_section_usage) + usemap_size(); +} + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +unsigned long __init section_map_size(void) +{ + return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); +} + +#else +unsigned long __init section_map_size(void) +{ + return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); +} + +struct page __init *__populate_section_memmap(unsigned long pfn, + unsigned long nr_pages, int nid, struct vmem_altmap *altmap, + struct dev_pagemap *pgmap) +{ + unsigned long size = section_map_size(); + + return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); +} +#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ + +void __weak __meminit vmemmap_populate_print_last(void) +{ +} + +static void *sparse_usagebuf __meminitdata; +static void *sparse_usagebuf_end __meminitdata; + +/* + * Helper function that is used for generic section initialization, and + * can also be used by any hooks added above. + */ +void __init sparse_init_early_section(int nid, struct page *map, + unsigned long pnum, unsigned long flags) +{ + BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end); + sparse_init_one_section(__nr_to_section(pnum), pnum, map, + sparse_usagebuf, SECTION_IS_EARLY | flags); + sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size(); +} + +static int __init sparse_usage_init(int nid, unsigned long map_count) +{ + unsigned long size; + + size = mem_section_usage_size() * map_count; + sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid); + if (!sparse_usagebuf) { + sparse_usagebuf_end = NULL; + return -ENOMEM; + } + + sparse_usagebuf_end = sparse_usagebuf + size; + return 0; +} + +static void __init sparse_usage_fini(void) +{ + sparse_usagebuf = sparse_usagebuf_end = NULL; +} + +/* + * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) + * And number of present sections in this node is map_count. + */ +static void __init sparse_init_nid(int nid, unsigned long pnum_begin, + unsigned long pnum_end, + unsigned long map_count) +{ + unsigned long pnum; + + if (sparse_usage_init(nid, map_count)) + panic("Failed to allocate usemap for node %d\n", nid); + + sparse_vmemmap_init_nid_early(nid); + + for_each_present_section_nr(pnum_begin, pnum) { + struct mem_section *ms; + unsigned long pfn = section_nr_to_pfn(pnum); + + if (pnum >= pnum_end) + break; + + ms = __nr_to_section(pnum); + if (!preinited_vmemmap_section(ms)) { + struct page *map; + + map = __populate_section_memmap(pfn, PAGES_PER_SECTION, + nid, NULL, NULL); + if (!map) + panic("Failed to allocate memmap for section %lu\n", pnum); + memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), + PAGE_SIZE)); + sparse_init_early_section(nid, map, pnum, 0); + } + } + sparse_usage_fini(); +} + +/* + * Allocate the accumulated non-linear sections, allocate a mem_map + * for each and record the physical to section mapping. + */ +void __init sparse_init(void) +{ + unsigned long pnum_end, pnum_begin, map_count = 1; + int nid_begin; + + /* see include/linux/mmzone.h 'struct mem_section' definition */ + BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); + memblocks_present(); + + if (compound_info_has_mask()) { + VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), + MAX_FOLIO_VMEMMAP_ALIGN)); + } + + pnum_begin = first_present_section_nr(); + nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); + + for_each_present_section_nr(pnum_begin + 1, pnum_end) { + int nid = sparse_early_nid(__nr_to_section(pnum_end)); + + if (nid == nid_begin) { + map_count++; + continue; + } + /* Init node with sections in range [pnum_begin, pnum_end) */ + sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); + nid_begin = nid; + pnum_begin = pnum_end; + map_count = 1; + } + /* cover the last node */ + sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); + sparse_init_subsection_map(); + vmemmap_populate_print_last(); +} |
