diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index e675a57145e3..55df688ea596 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -3,7 +3,7 @@ Date: Mar 2022 Contact: SJ Park Description: Interface for Data Access MONitoring (DAMON). Contains files for controlling DAMON. For more details on DAMON itself, - please refer to Documentation/admin-guide/mm/damon/index.rst. + please refer to Documentation/mm/damon/index.rst. What: /sys/kernel/mm/damon/admin/ Date: Mar 2022 @@ -173,6 +173,19 @@ Contact: SJ Park Description: Writing to and reading from this file sets and gets the per-probe attribute weight. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/preps/nr_preps +Date: Jun 2026 +Contact: SJ Park +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probing preparation action named '0' + to 'N-1' under the preps/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/preps//prep_action +Date: Jun 2026 +Contact: SJ Park +Description: Writing to and reading from this file sets and gets the probing + preparation action. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters/nr_filters Date: May 2026 Contact: SJ Park @@ -193,6 +206,20 @@ Description: If 'memcg' is written to the 'type' file, writing to and reading from this file sets and gets the path to the memory cgroup of the interest. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//min +Date: Sep 2026 +Contact: SJ Park +Description: If 'hugepage_size' is written to the 'type' file, writing to and + reading from this file sets and gets the minimum size of the + huge page of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//max +Date: Sep 2026 +Contact: SJ Park +Description: If 'hugepage_size' is written to the 'type' file, writing to and + reading from this file sets and gets the maximum size of the + huge page of the interest. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//matching Date: May 2026 Contact: SJ Park diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index bd8c80f3bbc9..7fe950425216 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1321,6 +1321,10 @@ All memory amounts are in bytes. If a value which is not aligned to PAGE_SIZE is written, the value may be rounded up to the closest PAGE_SIZE multiple when read back. +For the limit files described below, an empty or all-whitespace +write is accepted and sets the limit to 0. To disable a limit, +write "max"; to set it to zero explicitly, write "0". + memory.current A read-only single value file which exists on non-root cgroups. diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst index 7663c610fe90..5f1df6d08050 100644 --- a/Documentation/admin-guide/kdump/vmcoreinfo.rst +++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst @@ -325,7 +325,7 @@ NR_FREE_PAGES On linux-2.6.21 or later, the number of free pages is in vm_stat[NR_FREE_PAGES]. Used to get the number of free pages. -PG_lru|PG_private|PG_swapcache|PG_swapbacked|PG_hwpoison|PG_head_mask +PG_lru|PG_swapcache|PG_swapbacked|PG_hwpoison|PG_head_mask -------------------------------------------------------------------------- Page attributes. These flags are used to filter various unnecessary for diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index da5f9afd08ae..ba4725544856 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -74,8 +74,11 @@ comma (","). │ │ │ │ │ │ nr_regions/min,max │ │ │ │ │ │ :ref:`probes `/nr_probes │ │ │ │ │ │ │ 0/weight + │ │ │ │ │ │ │ │ preps/nr_preps + │ │ │ │ │ │ │ │ │ 0/prep_action + │ │ │ │ │ │ │ │ │ ... │ │ │ │ │ │ │ │ filters/nr_filters - │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path + │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path,min,max │ │ │ │ │ │ │ │ │ ... │ │ │ │ │ │ │ ... │ │ │ │ │ :ref:`targets `/nr_targets @@ -283,9 +286,18 @@ In the beginning, this directory has only one file, ``nr_probes``. Writing a number (``N``) to the file creates the number of child directories named ``0`` to ``N-1``. Each directory represents each monitoring probe. -In each probe directory, one directory, ``filters`` exists. The directory -contains files for installing filters for the probe, that is used to determine -the data attribute for the probe. +In each probe directory, two directories, ``preps`` and ``filters`` exist. The +directories contain files for installing probing preparation actions and +filters for the probe, that are used to determine the data attribute for the +probe. + +In the beginning, ``preps`` directory has only one file, ``nr_preps``. +Writing a number (``N``) to the file creates the number of child directories +named ``0`` to ``N-1``. Each directory represents each preparation action. +Each directory has one file, ``prep_action``. The preparation action can be +selected by writing the name of the action to the ``prep_action`` file. Refer +to the :ref:`design doc ` for the list of +supported actions. Each probe directory also contains ``weight`` file. Reading from and writing to the file gets and sets the :ref:`attributes-only monitoring @@ -296,6 +308,8 @@ Writing a number (``N``) to the file creates the number of child directories named ``0`` to ``N-1``. Each directory represents each filter and works in a way similar to that for :ref:`DAMOS filter `. When the filter ``type`` is ``memcg``, ``path`` file acts as ``memcg_path`` for :ref:`DAMOS +filter `. When the filter ``type`` is ``hugepage_size``, +``min`` and ``max`` files act as files of the same names for :ref:`DAMOS filter `. .. _sysfs_targets: @@ -539,6 +553,10 @@ and ``damon_target_idx``. To ``type`` file, you can write the type of the filter. Refer to :ref:`the design doc ` for available type names, their meaning and on what layer those are handled. +For ``probe_hits_wsum`` type, you can specify the minimum and maximum probe +hits weighted sum value for the filter to ``min`` and ``max`` files, +respectively. + For ``memcg`` type, you can specify the memory cgroup of the interest by writing the path of the memory cgroup from the cgroups mount point to ``memcg_path`` file. For ``addr`` type, you can specify the start and end diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst index ad8e7a41f3b5..c329ca747b8c 100644 --- a/Documentation/admin-guide/mm/ksm.rst +++ b/Documentation/admin-guide/mm/ksm.rst @@ -174,7 +174,7 @@ advisor_mode The section about ``advisor`` explains in detail how the scan time advisor works. -adivsor_max_cpu +advisor_max_cpu specifies the upper limit of the cpu percent usage of the ksmd background thread. The default is 70. @@ -183,10 +183,10 @@ advisor_target_scan_time pages. The default value is 200 seconds. advisor_min_pages_to_scan - specifies the lower limit of the ``pages_to_scan`` parameter of the - scan time advisor. The default is 500. + specifies the initial value of the ``pages_to_scan`` parameter of + the scan time advisor. The default is 500. -adivsor_max_pages_to_scan +advisor_max_pages_to_scan specifies the upper limit of the ``pages_to_scan`` parameter of the scan time advisor. The default is 30000. diff --git a/Documentation/arch/powerpc/vmemmap_dedup.rst b/Documentation/arch/powerpc/vmemmap_dedup.rst index dc4db59fdf87..8286acbca9bc 100644 --- a/Documentation/arch/powerpc/vmemmap_dedup.rst +++ b/Documentation/arch/powerpc/vmemmap_dedup.rst @@ -19,82 +19,28 @@ With 1G PUD level mapping, we require 16384 struct pages and a single 64K vmemmap page can contain 1024 struct pages (64K/sizeof(struct page)). Hence we require 16 64K pages in vmemmap to map the struct page for 1G PUD level mapping. -Here's how things look like on device-dax after the sections are populated:: - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PUD | +-----------+ | | | - | level | | . | ----------------------+ | | - | mapping | +-----------+ | | - | | | . | ------------------------+ | - | | +-----------+ | - | | | 15 | --------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ - - With 4K page size, 2M PMD level mapping requires 512 struct pages and a single 4K vmemmap page contains 64 struct pages(4K/sizeof(struct page)). Hence we require 8 4K pages in vmemmap to map the struct page for 2M pmd level mapping. -Here's how things look like on device-dax after the sections are populated:: +Here's how things look on device-dax after vmemmap-optimized sections are +populated. ``N`` is the number of vmemmap pages required by the DAX mapping +above:: - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PMD | +-----------+ | | | - | level | | 5 | ----------------------+ | | - | mapping | +-----------+ | | - | | | 6 | ------------------------+ | - | | +-----------+ | - | | | 7 | --------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ - -With 1G PUD level mapping, we require 262144 struct pages and a single 4K -vmemmap page can contain 64 struct pages (4K/sizeof(struct page)). Hence we -require 4096 4K pages in vmemmap to map the struct pages for 1G PUD level -mapping. - -Here's how things look like on device-dax after the sections are populated:: - - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PUD | +-----------+ | | | - | level | | . | ----------------------+ | | - | mapping | +-----------+ | | - | | | . | ------------------------+ | - | | +-----------+ | - | | | 4095 | --------------------------+ - | | +-----------+ + Device DAX vmemmap pages (N pages) backing page frames + +-----------+ ---virt_to_page---> +-----------+ mapping to +-------------+ + | | | 0 | -------------> | 0 | + | | +-----------+ +-------------+ + | | | 1 | ------+ + | | +-----------+ | + | | | 2 | ------+ + | | +-----------+ | + | | | . | ------+ +-------------+ + | PMD/PUD | +-----------+ | | A single, | + | level | | . | ------+------> | per-zone | + | mapping | +-----------+ | | shared tail | + | | | N - 1 | ------+ | page | + | | +-----------+ +-------------+ | | | | | | diff --git a/Documentation/core-api/memory-allocation.rst b/Documentation/core-api/memory-allocation.rst index 0f19dd524323..648222b6c857 100644 --- a/Documentation/core-api/memory-allocation.rst +++ b/Documentation/core-api/memory-allocation.rst @@ -19,10 +19,18 @@ Diversity of the allocation APIs combined with the numerous GFP flags makes the question "How should I allocate memory?" not that easy to answer, although very likely you should use +:: + + kzalloc_obj(); + +if you need memory for an object and + :: kzalloc(, GFP_KERNEL); +if you need memory for a buffer. + Of course there are cases when other allocation APIs and different GFP flags must be used. @@ -139,10 +147,13 @@ allocate memory for an array, there are kmalloc_array() and kcalloc() helpers. The helpers struct_size(), array_size() and array3_size() can be used to safely calculate object sizes without overflowing. -The maximal size of a chunk that can be allocated with `kmalloc` is -limited. The actual limit depends on the hardware and the kernel -configuration, but it is a good practice to use `kmalloc` for objects -smaller than page size. +Since 7.0 there are type aware kmalloc-family helpers that let you safely and +conveniently allocate a single object or arrays of objects with kzalloc_obj() +and kmalloc_obj() and their array versions kzalloc_objs() and +kmalloc_objs(). These helpers only need the type of the object that should be +allocated and the count of elements in the array for the array versions. + +As of v7.2, vast majority of the memory allocations use kzalloc_obj(). The address of a chunk allocated with `kmalloc` is aligned to at least ARCH_KMALLOC_MINALIGN bytes. For sizes which are a power of two, the @@ -154,9 +165,16 @@ Chunks allocated with kmalloc() can be resized with krealloc(). Similarly to kmalloc_array(): a helper for resizing arrays is provided in the form of krealloc_array(). -For large allocations you can use vmalloc() and vzalloc(), or directly -request pages from the page allocator. The memory allocated by `vmalloc` -and related functions is not physically contiguous. +`kmalloc` always allocates physically contiguous memory and the maximal size of +a chunk that can be allocated with `kmalloc` is limited by `KMALLOC_MAX_SIZE`, +which matches the page allocator's MAX_PAGE_ORDER limit. + +Internally, the slab allocator differentiates allocations of different orders +and delegates larger allocations to the page allocator, but for the users of +`kmalloc` family it is entirely transparent. + +For large allocations that do not require physically contiguous memory you can +use vmalloc() and vzalloc() family. If you are not sure whether the allocation size is too large for `kmalloc`, it is possible to use kvmalloc() and its derivatives. It will diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index c1d03a5a2a19..e135b61bc87d 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -52,6 +52,15 @@ Virtually Contiguous Mappings .. kernel-doc:: mm/vmalloc.c :export: +Kernel Page Table Permissions +============================= + +.. kernel-doc:: include/linux/set_memory.h + :doc: Kernel page table permissions + +.. kernel-doc:: include/linux/set_memory.h + :internal: + File Mapping and Page Cache =========================== diff --git a/Documentation/core-api/pin_user_pages.rst b/Documentation/core-api/pin_user_pages.rst index c16ca163b55e..e0acedbd1d48 100644 --- a/Documentation/core-api/pin_user_pages.rst +++ b/Documentation/core-api/pin_user_pages.rst @@ -226,15 +226,6 @@ will be pinned longterm, and whose data will be accessed. Unit testing ============ -This file:: - - tools/testing/selftests/mm/gup_test.c - -has the following new calls to exercise the new pin*() wrapper functions: - -* PIN_FAST_BENCHMARK (./gup_test -a) -* PIN_BASIC_TEST (./gup_test -b) - You can monitor how many total dma-pinned pages have been acquired and released since the system was booted, via two new /proc/vmstat entries: :: diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst index d1b690b17169..c0d329372342 100644 --- a/Documentation/dev-tools/kmemleak.rst +++ b/Documentation/dev-tools/kmemleak.rst @@ -66,9 +66,14 @@ Memory scanning parameters can be modified at run-time by writing to the Kmemleak can also be disabled at boot-time by passing ``kmemleak=off`` on the kernel command line. -Memory may be allocated or freed before kmemleak is initialised and -these actions are stored in an early log buffer. The size of this buffer -is configured via the CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE option. +Memory may be allocated or freed before kmemleak is initialised, so a +static pool of metadata objects is used to track those allocations. Once +kmemleak is fully initialised the pool becomes an emergency reserve, used +whenever a metadata object cannot be allocated from the slab. The number +of objects in the pool is configured via the +CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE option. Exhausting it at run time +prints "Cannot allocate a kmemleak_object structure" and disables +kmemleak. If CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF are enabled, the kmemleak is disabled by default. Passing ``kmemleak=on`` on the kernel command @@ -185,7 +190,8 @@ reported by kmemleak because values found during the memory scanning point to such objects. To reduce the number of false negatives, kmemleak provides the kmemleak_ignore, kmemleak_scan_area, kmemleak_no_scan and kmemleak_erase functions (see above). The task stacks also increase the -amount of false negatives and their scanning is not enabled by default. +amount of false negatives and their scanning is enabled by default; it +can be turned off with ``stack=off``. The false positives are objects wrongly reported as being memory leaks (orphan). For objects known not to be leaks, kmemleak provides the @@ -195,12 +201,12 @@ longer be scanned. Some of the reported leaks are only transient, especially on SMP systems, because of pointers temporarily stored in CPU registers or -stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing +stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 5000) representing the minimum age of an object to be reported as a memory leak. The ``min_unref_scans`` module parameter requires an object to be seen unreferenced in that many consecutive scans before it is reported. It -defaults to 2 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the +defaults to 3 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the periodic scan thread confirms a leak on its own, and to 1 otherwise. A value of 1 preserves the historical behaviour; higher values filter the transient false positives described above, at the cost of delaying genuine @@ -212,8 +218,10 @@ Limitations and Drawbacks ------------------------- The main drawback is the reduced performance of memory allocation and -freeing. To avoid other penalties, the memory scanning is only performed -when the /sys/kernel/debug/kmemleak file is read. Anyway, this tool is +freeing. To avoid other penalties, the memory scanning is performed by a +periodic thread rather than on every allocation. Reading the +/sys/kernel/debug/kmemleak file only lists the objects found by the last +scan; writing ``scan`` to it triggers a new one. Anyway, this tool is intended for debugging purposes where the performance might not be the most important requirement. diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index 99495497ea6d..1824462a4371 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -649,8 +649,8 @@ Writeback. The first can be used independently to the others. The VM can try to release clean pages in order to reuse them. To do this it can call -->release_folio on clean folios with the private -flag set. Clean pages without PagePrivate and with no external references +->release_folio on clean folios with folio->private set. Clean pages +without folio->private set and with no external references will be released without notice being given to the address_space. To achieve this functionality, pages need to be placed on an LRU with @@ -674,7 +674,7 @@ filemap_fdatawait_range, to wait for all writeback to complete. An address_space handler may attach extra information to a page, typically using the 'private' field in the 'struct page'. If such -information is attached, the PG_Private flag should be set. This will +information is attached, non-NULL 'private' field will cause various VM routines to make extra calls into the address_space handler to deal with that data. diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index aed6cb1cf483..e82390e77a70 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -249,7 +249,7 @@ For each ``aggregation interval``, it compares the access frequencies sum of the two regions' sizes is smaller than the size of total regions divided by the ``minimum number of regions``, DAMON merges the two regions. If the resulting number of total regions is still higher than ``maximum number of -regions``, it repeats the merging with increasing access frequenceis difference +regions``, it repeats the merging with increasing access frequencies difference threshold until the upper-limit of the number of regions is met, or the threshold becomes higher than possible maximum value (``aggregation interval`` divided by ``sampling interval``). Then, after it reports and clears the @@ -293,8 +293,15 @@ registration is made by specifying a probe per attribute. Each of the probe specifies a rule to determine if a given memory region has the related attribute. The rule is constructed with multiple filters. The filters work same to :ref:`DAMOS filters ` except the supported -filter types. Currently only ``anon`` and ``memcg`` filter types are supported -for data attributes monitoring. +filter types. Currently below filter types are supported. + +- ``anon``: Same to that for DAMOS filters. +- ``memcg``: Same to that for DAMOS filters. +- ``pgidle_unset``: Matches if the page for the memory is marked as not + access-idle. +- ``pgidle_set``: Matches if the page for the memory is marked as access-idle. +- ``hugepage_size``: Matches if the page for the memory is a part of a hugepage + of a given size range. If such probes are registered, DAMON executes the probes for each region's sampling memory when it does the access :ref:`sampling @@ -305,6 +312,13 @@ Users can therefore know how much of a given DAMON region has a specific data attribute by reading the per-region per-probe probe hits counter after each aggregation interval. +Users can optionally register probing preparation actions per probe. If such +actions are registered, DAMON applies the actions to each region's sampling +memory before starting the next sampling interval. Currently only one action, +``set_pgidle`` is supported. The action marks the page for the probing target +memory as access-idle. This can be useful to be used together with +``pgidle_unset`` or ``pgidle_set`` probe filter. + This is a sampling based mechanism. Hence, it is lightweight but the output may include some measurement errors. The output should be used with good understanding of statistics. @@ -431,7 +445,7 @@ users to set the aimed amount of access events to observe via DAMON within given time interval. The target can be specified by the user as a ratio of DAMON-observed access events to the theoretical maximum amount of the events (``access_bp``) that measured within a given number of aggregations -(``aggrs``). +(``aggrs``). The ratio is in basis point (bp or 1/10,000). The DAMON-observed access events are calculated in byte granularity based on DAMON :ref:`region assumption `. For @@ -682,9 +696,11 @@ There are two such tuning algorithms that users can select as they need. This is the default selection. If unsure, use this. - ``temporal``: More straightforward algorithm. Tries to achieve the goal as fast as possible, using maximum allowed quota, but only for a temporal short - time. When the quota is under-achieved, this algorithm keeps tuning quota to - a maximum allowed one. Once the quota is [over]-achieved, this sets the - quota zero. Useful for deterministic control required environments. + time. When the goal is under-achieved, this algorithm keeps tuning quota to + a maximum allowed one. Once the goal is [over-]achieved, this sets the + quota zero. Useful for deterministic control required environments. Note + that the zero quota is a valid quota, and therefore ``qt_exceeds`` :ref:`stat + ` will keep increasing in this case. The goal can be specified with five parameters, namely ``target_metric``, ``target_value``, ``current_value``, ``nid`` and ``path``. The auto-tuning @@ -701,7 +717,8 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to in microseconds that measured from last quota reset to next quota reset. DAMOS does the measurement on its own, so only ``target_value`` need to be set by users at the initial time. In other words, DAMOS does self-feedback. -- ``node_mem_used_bp``: Specific NUMA node's used memory ratio in bp (1/10,000). +- ``node_mem_used_bp``: Specific NUMA node's used memory ratio in basis point + (bp or 1/10,000). - ``node_mem_free_bp``: Specific NUMA node's free memory ratio in bp (1/10,000). - ``node_memcg_used_bp``: Specific cgroup's node used memory ratio for a specific NUMA node, in bp (1/10,000). @@ -713,6 +730,8 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to bp (1/10,000). - ``node_eligible_mem_bp``: Scheme target access pattern-eligible memory ratio of a node in bp (1/10,000). +- ``hugepage_mem_bp``: Total huge page to total used memory ratio in bp + (1/10,000). ``nid`` is optionally required for ``node_mem_used_bp``, ``node_mem_free_bp``, ``node_memcg_used_bp``, ``node_memcg_free_bp`` and ``node_eligible_mem_bp`` to @@ -809,6 +828,9 @@ Below ``type`` of filters are currently supported. - Applied to pages that belonging to a given address range. - target - Applied to pages that belonging to a given DAMON monitoring target. + - probe_hits_wsum + - Matches to monitoring regions having a given range of :ref:`probe + hits weighted sum ` value. - Operations layer handled, supported by only ``paddr`` operations set. - anon - Applied to pages that containing data that not stored in files. @@ -846,8 +868,8 @@ scheme's execution. - ``nr_applied``: Total number of regions that the scheme is applied. - ``sz_applied``: Total size of regions that the scheme is applied. - ``qt_exceeds``: Total number of times the quota of the scheme has exceeded. -- ``nr_snapshots``: Total number of DAMON snapshots that the scheme is tried to - be applied. +- ``nr_snapshots``: Total number of DAMON snapshots that the scheme is + completely tried to be applied. - ``max_nr_snapshots``: Upper limit of ``nr_snapshots``. "A scheme is tried to be applied to a region" means DAMOS core logic determined @@ -871,11 +893,12 @@ action is ``pageout`` while all pages of the region are unreclaimable, applying the action to the region will fail. Unlike normal stats, ``max_nr_snapshots`` is set by users. If it is set as -non-zero and ``nr_snapshots`` be same to or greater than ``nr_snapshots``, the -scheme is deactivated. +non-zero and ``nr_snapshots`` equals or is greater than ``max_nr_snapshots``, +the scheme is deactivated. Note that, unlike watermarks, even if a scheme's +``nr_snapshots`` reaches ``max_nr_snapshots``, monitoring will not stop. To know how user-space can read the stats via :ref:`DAMON sysfs interface -`, refer to :ref:s`stats ` part of the +`, refer to :ref:`stats ` part of the documentation. Regions Walking diff --git a/Documentation/mm/damon/maintainer-profile.rst b/Documentation/mm/damon/maintainer-profile.rst index fb2fa00cc9aa..a7c135233937 100644 --- a/Documentation/mm/damon/maintainer-profile.rst +++ b/Documentation/mm/damon/maintainer-profile.rst @@ -106,18 +106,9 @@ AI Review For patches that are publicly posted to DAMON mailing list (damon@lists.linux.dev), AI reviews of the patches will be available at -sashiko.dev. The reviews could also be sent as mails to the author of the -patch. +sashiko.dev. The reviews will also be sent as replies to the author of the +patch and the mailing list. -Patch authors are encouraged to check the AI reviews and share their opinions. -The sharing could be done as a reply to the mail thread. Consider reducing the -recipients list for such sharing, since some people are not really interested -in AI reviews. As a rule of thumb, drop stable@vger.kernel.org and individuals -except DAMON maintainer. - -`hkml` also provides a `feature -`_ -for such sharing. Please feel free to use the feature. - -It is only an optional recommendation. DAMON maintainer could also ask any -question about the AI reviews, though. +Patch authors are encouraged to check the AI reviews and share their opinions +by replying on the mail thread. It is only an optional recommendation. DAMON +maintainer could also ask any question about the AI reviews, though. diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 13a79f5d092c..e9ae5cd82163 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -25,6 +25,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`. physical_memory page_tables + kernel-page-tables process_addrs bootmem page_allocation diff --git a/Documentation/mm/kernel-page-tables.rst b/Documentation/mm/kernel-page-tables.rst new file mode 100644 index 000000000000..c16de34b4f79 --- /dev/null +++ b/Documentation/mm/kernel-page-tables.rst @@ -0,0 +1,410 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================== +Kernel Page Tables +================== + +Introduction +============ + +The kernel page tables are created early during boot and, unlike the page +tables of user processes, most of them remain static throughout the system +lifetime. + +Every architecture has a direct map (also called linear map) that maps the +physical memory at a fixed offset, so that a physical address can be +translated to a kernel virtual address with simple arithmetic. On most +architectures the direct map is a part of the kernel page tables, with a few +exceptions described in the `Direct map`_ section below. + +On 32-bit systems with high memory the direct map covers only a part of the +physical memory, see Documentation/mm/highmem.rst. + +The vmalloc area, present on every architecture with an MMU, is used for +allocations of virtually contiguous memory whose backing pages are not +necessarily physically contiguous, and for mapping of the device memory. Its +page tables are created and torn down at runtime, see +Documentation/mm/vmalloc.rst. + +Architectures that use the `SPARSEMEM_VMEMMAP` memory model reserve a range of +kernel address space for the memory map, so that `struct page` objects appear +as a virtually contiguous array indexed by the page frame number, see +Documentation/mm/memory-model.rst. + +Besides these, the kernel image may be mapped in a dedicated part of the +kernel address space rather than accessed through the direct map. In that +case its mapping is an alias of the direct map of the physical memory the +image occupies. + +The rest of the kernel address space is architecture specific. For instance, +x86 has a region for the EFI runtime services and s390 has a region for the +code that has to run in the 31-bit addressing mode. + +Direct map +========== + +On most architectures the direct map is an ordinary part of the kernel page +tables. It is created early during boot with the largest pages the hardware +and the kernel configuration allow. + +Several architectures are different. + +MIPS +---- + +MIPS does not map the physical memory with page tables at all. Instead, a +part of the kernel virtual address space is a window into the physical +address space: the hardware translates the addresses that fall into that +window by a fixed transformation of the address bits, without walking the +page tables and without using the TLB. The memory attributes, such as +cacheability and the privilege level required to access the memory, are a +property of the window rather than of an individual page. + +There are no page table entries describing the direct map, so its properties +cannot be changed for an individual page. On 32-bit systems the window covers +only 512 MiB of the physical memory, so everything above that is high memory. + +LoongArch +--------- + +Like MIPS, LoongArch maps the physical memory with a hardware window. The +window covers 256 TiB of the physical address space on 64-bit and 512 MiB on +32-bit systems, and everything above that is high memory. + +The window occupies the lower part of the kernel address space. The upper +part, which includes the vmalloc area, is mapped with kernel page tables. + +PowerPC with the hash MMU +------------------------- + +On 64-bit PowerPC systems with the hash MMU the direct map does not exist in +the Linux page tables. It is installed into the hardware hash page table early +during boot. + +Modifying such mappings requires updating the hash page table directly, and +the hash MMU code implements this only for the kernel image permissions, +`debug_pagealloc` and KFENCE. + +With the radix MMU the direct map is a part of the ordinary kernel page +tables. + +Modifying the kernel page tables +================================ + +Except for the vmalloc area, the kernel page tables are mostly static. Still, +there are cases when the permissions of existing kernel mappings have to be +updated, for instance when a module is loaded and its text becomes read-only +and executable, or when a page is temporarily removed from the direct map to +reduce its exposure. + +There are two families of functions for this, both declared in +`include/linux/set_memory.h`: + +* `set_memory_*()` change permissions of an arbitrary kernel mapping. They + take a kernel virtual address and the number of pages. + +* `set_direct_map_*()` change permissions of the direct mapping for the range + of page frames starting at the page represented by a `struct page`. They take + a `struct page` pointer and the number of pages. + +Architectures that implement `set_memory()` select `CONFIG_ARCH_HAS_SET_MEMORY` + +Architectures that implement `set_direct_map()` select +`CONFIG_ARCH_HAS_SET_DIRECT_MAP`. + +Common semantics +---------------- + +Ranges +~~~~~~ + +The `set_memory()` functions expect a range described by a start address and a +number of pages. The address must be page aligned and the entire range must be +covered by page table entries the architecture knows how to update. + +The entries may be marked as not present, set_memory_p() and set_memory_valid() +exist exactly to bring such a mapping back. + +When a range does not qualify, an architecture will usually say so by returning +an error and sometimes by a WARN()ing as well, unless it prefers to keep it to +itself and return success, see `Architecture specific differences`_. + +The `set_direct_map()` functions expect a range described by the first +`struct page` and a number of pages, and they update the direct map starting +at that page. + +The pages that follow the first one are updated regardless of what they are, so +the caller has to make sure that the range does not extend beyond the memory it +owns. + +Some architectures cannot split a large mapping, and they reject a range that +is a part of one, see `Architecture specific differences`_. + +Calling `set_memory()` with the number of pages set to zero is a no-op that +returns success, except on arm64, where doing nothing to the wrong address is +still an error. + +Aliases +~~~~~~~ + +A physical page may be mapped several times, for instance in the direct map +and in the vmalloc area, and the permissions of these mappings may differ. + +Whether the direct map alias is updated by a `set_memory()` call, and +which permission bits make it there, is entirely up to the architecture, and +there is not much agreement between them, see +`Architecture specific differences`_. + +Relying on that is a gamble; code that needs the direct map alias to change +should say so with the `set_direct_map()` APIs. + +Failures and partial updates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Both families of functions return 0 on success and a negative error code on +failure. The most common failures are `-EINVAL` for a range that cannot be +handled and `-ENOMEM` when splitting a large mapping fails to allocate a page +table. + +Some of the range checks happen upfront, so their failure leaves the page +tables unchanged. + +**The update is not atomic and there is no rollback.** + +The architecture implementations walk the range and update the page tables as +they go, and they stop at the first entry that cannot be updated. When an error +is returned, an arbitrary prefix of the range may have been updated already, +and the same is true for the direct map alias when the architecture updates it. + +For example, when a range spans two large mappings and splitting the second +one fails because there is no memory for a page table, the first one is +already split and updated by the time the error is returned. + +None of the APIs inform the caller where in the range they failed, so reverting +such a partial update is possible in principle but unreliable in practice. + +The caller may try to restore the original permissions over the entire range, +but that revert goes through the very code that has just failed, which does +not inspire much confidence. + +The callers should therefore be prepared to give up on the memory in question: +leak it or panic, but never return it to the allocator before the permissions +are restored and never assume that the requested permissions are in effect. +Neither option is appealing, but both beat handing out a page whose +permissions nobody knows. + +TLB flushing +~~~~~~~~~~~~ + +The `set_memory()` functions flush the TLB for the affected range before +they return, so that the new permissions are in effect for every CPU. + +The `set_direct_map()` functions have `_noflush` in their names because when +they were first introduced on x86, the intention was that the TLB flushing +could be optimized by letting the caller handle it. + +For example, vfree() batches the TLB flushes for the areas allocated with +`VM_FLUSH_RESET_PERMS`, folding the flush of the direct map into the flush it +has to do for the vmalloc mapping anyway. + +Some architectures flush the TLB in the `_noflush` functions anyway, so the +name is best read as a suggestion. It does not make the flush by the caller +unnecessary, it only makes it more expensive. + +A caller that changes the permissions to more restrictive ones must flush the +TLB itself. + +Context +~~~~~~~ + +Architectures use different locking mechanisms to synchronize kernel page table +updates, and both families of functions may sleep, for instance when they +allocate memory to split a large mapping. + +The caller cannot presume it is safe to call these APIs from an atomic context. + +The `set_direct_map()` functions must not be called for high memory pages, +which have no direct map alias to update. + +Unimplemented APIs silently succeed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When an architecture does not implement these APIs, the generic stubs in +`include/linux/set_memory.h` return 0, that is, they report success for work +they have no intention of doing. + +The same happens inside several architecture implementations. The arm64 +`set_direct_map()` functions return 0 when can_set_direct_map() is false, and +the LoongArch `set_memory()` functions return 0 for the addresses in its +windowed direct mapping, which is not backed by page tables at all. + +Returning success without actually updating the page tables is a deliberate +trade-off that keeps the callers free of `#ifdef`\ s, but they have to realize: + +**a return value of 0 does not imply that the permissions were actually +changed.** + +For best-effort hardening that is good enough. When correctness or security +depends on the permissions, the caller has to make sure the architecture +really implements what it needs. For instance, secretmem depends on +`CONFIG_ARCH_HAS_SET_DIRECT_MAP` and calls can_set_direct_map() at runtime. + +Architecture specific differences +================================= + +The APIs are implemented by seven architectures and, beyond the common +semantics described above, their behaviour differs in several respects. + +Which of the APIs are implemented: + +========= ===================== ========================= +Arch `ARCH_HAS_SET_MEMORY` `ARCH_HAS_SET_DIRECT_MAP` +========= ===================== ========================= +arm yes no +arm64 yes yes +loongarch yes yes +powerpc yes no +riscv yes (MMU only) yes (MMU only) +s390 yes yes +x86 yes yes +========= ===================== ========================= + +Only set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() are +available everywhere, and even these are not universal: some architectures +restrict the ranges that can be modified, for instance arm64 rejects direct map +addresses and accepts only addresses in vmalloc space. The architectures that +may run on hardware without an execute permission bit, like x86 and s390, +silently skip the update of the executable bit. + +set_memory_rox() has a generic implementation that calls set_memory_ro() and +set_memory_x() in turn; PowerPC, s390 and x86 override it with a single-pass +version. + +Several architectures define additional APIs. Some of those may have identical +semantics but different names. For example, making a mapping present or not +present is spelled differently: set_memory_p() and set_memory_np() on x86 and +PowerPC, set_memory_valid() on arm and arm64. + +The direct map and the kernel image are normally mapped with the largest +possible pages, and changing the permissions of a single page inside such a +mapping requires splitting it, which not every architecture can do. + +arm +--- + +* Does not implement `set_direct_map()`. +* Provides set_memory_valid(). +* set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() accept + only vmalloc and module addresses. +* set_memory_valid() accepts any address. +* Does not update mapping aliases. + +arm64 +----- + +* Provides set_memory_valid(). +* Provides the memory encryption helpers, which are effective only when the + kernel runs as a confidential guest. +* set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() accept + only vmalloc and module addresses: + + - the range must fit in the VM area that contains its start + - the VM area must have `VM_ALLOC` set and `VM_ALLOW_HUGE_VMAP` clear + +* set_memory_valid() accepts any address. +* The encryption helpers accept only direct map addresses. +* The `set_memory()` functions propagate the read-only and the read-write + changes to the direct map alias when `rodata=full` is in effect. +* Splits leaf mappings before the update on the hardware that supports it. + The split itself may be partial when it fails midway, but the permissions are + left untouched in that case. + Without support for splitting large mappings, a range that covers a leaf + entry only partially fails with a WARN()ing and `-EINVAL`. + If a range spans one or more full leaf entries and a partial leaf entry, the + permissions of the full leaf entries are updated before the failure. +* The `set_direct_map()` functions return 0 without doing anything when the + direct map cannot be modified, see can_set_direct_map(). +* Skips the TLB flush in `set_memory()` when the update only turns an invalid + mapping into a valid one. +* Does not flush TLB in `set_direct_map()`. + +LoongArch +--------- + +* Accepts only the addresses above the hardware window and silently returns + success for the rest, see `Direct map`_. +* Does not update mapping aliases. +* Does not split anything: a leaf entry is updated as a whole, which changes + the permissions of the entire large mapping. +* Flushes the TLB in `set_direct_map()`. + +PowerPC +------- + +* Does not implement `set_direct_map()`. +* Provides set_memory_np() and set_memory_p(). +* Rejects huge vmalloc mappings. +* With the hash MMU on 64-bit systems accepts nothing but the vmalloc and the + I/O regions. +* With the radix MMU accepts direct map addresses, but still cannot split a + large mapping. +* Does not update mapping aliases. + +riscv +----- + +* Implements both APIs only when the MMU is enabled. +* Provides set_memory_rw_nx(). +* The `set_memory()` functions accept any mapped kernel address, including the + direct map, but a vmalloc range must have the `pages` array of its VM area + populated, which rules out vmap() and ioremap() mappings. +* On 64-bit systems the `set_memory()` functions update the direct map alias of + a vmalloc range, including the executable bit. +* Does not split vmalloc ranges: a leaf entry is updated as a whole, which + changes the permissions of the entire large mapping. +* Splits the direct map on 64-bit systems. +* Flushes the TLB in `set_direct_map()`. + +s390 +---- + +* Provides set_memory_4k(), set_memory_rwnx() and the + `__set_memory_*(start, end)` variants that take a range rather than a page + count. +* The `set_memory()` functions accept any mapped kernel address, including the + direct map. +* Skips the update of the executable bit when the hardware has no support for + it. +* The `set_memory()` functions propagate only the read-only and read-write + changes to the direct map alias of a `VM_ALLOC` area, and deliberately not + the executable bit. +* Splits leaf PUD and PMD entries when the range is not aligned to them or when + set_memory_4k() is requested. +* Updates the page table entries with instructions that invalidate the + corresponding TLB entries, so no separate flush is needed anywhere. + +x86 +--- + +* Provides the largest set of operations on top of the common ones: + + - the cache attribute helpers: set_memory_uc(), set_memory_wc(), + set_memory_wb() + - presence control: set_memory_np() and set_memory_p() + - set_memory_4k() + - set_memory_global() and set_memory_nonglobal() + - the array variants that operate on `struct page` arrays or arrays of + virtual addresses + - memory encryption: set_memory_encrypted() and set_memory_decrypted() + +* The `set_memory()` functions accept any mapped kernel address, including the + direct map, and silently succeed for the unmapped holes inside it. +* Does nothing in set_memory_x() and set_memory_nx() when the CPU has no + execute permission bit. +* The `set_memory()` functions apply the change to the direct map alias and, + for the kernel image, to the high kernel mapping. The NX bit is never + propagated, so that the direct map stays non-executable. +* Splits large mappings on demand and can collapse them back when the + permissions become uniform again. +* Does not flush the TLB in `set_direct_map()`, but splitting a large + mapping flushes it anyway. diff --git a/Documentation/mm/ksm.rst b/Documentation/mm/ksm.rst index 2b4f72f1f953..5a80573162de 100644 --- a/Documentation/mm/ksm.rst +++ b/Documentation/mm/ksm.rst @@ -78,7 +78,7 @@ The frequency of such scans is defined by Reference --------- .. kernel-doc:: mm/ksm.c - :functions: mm_slot ksm_scan stable_node rmap_item + :functions: ksm_mm_slot ksm_scan ksm_stable_node ksm_rmap_item -- Izik Eidus, diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst index a6bd3fe6423a..bd027377dff6 100644 --- a/Documentation/mm/page_owner.rst +++ b/Documentation/mm/page_owner.rst @@ -199,6 +199,7 @@ Usage -p Sort by pid. -P Sort by tgid. -n Sort by task command name. + -M Sort by module name. -r Sort by memory release time. -s Sort by stack trace. -t Sort by times (default). @@ -240,8 +241,10 @@ Usage group ID numbers appear in . --name Select by task command name. This selects the blocks whose task command name appear in . + --module Select by module name. This selects the blocks whose + module name appear in . - , , are single arguments in the form of a comma-separated list, + , , , are single arguments in the form of a comma-separated list, which offers a way to specify individual selecting rules. @@ -249,6 +252,7 @@ Usage ./page_owner_sort --pid=1 ./page_owner_sort --tgid=1,2,3 ./page_owner_sort --name name1,name2 + ./page_owner_sort --module xfs,ext4 STANDARD FORMAT SPECIFIERS ========================== @@ -265,6 +269,7 @@ STANDARD FORMAT SPECIFIERS ft free_ts timestamp of the page when it was released at alloc_ts timestamp of the page when it was allocated ator allocator memory allocator for pages + mod module kernel module name For --cull option: @@ -275,6 +280,7 @@ STANDARD FORMAT SPECIFIERS f free whether the page has been released or not st stacktrace stack trace of the page allocation ator allocator memory allocator for pages + mod module kernel module name Filtering page_owner output ============================ diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst index a7296f251799..78231995e490 100644 --- a/Documentation/mm/process_addrs.rst +++ b/Documentation/mm/process_addrs.rst @@ -537,6 +537,17 @@ We establish basic locking rules when interacting with page tables: * When changing a page table entry the page table lock for that page table **must** be held, except if you can safely assume nobody can access the page tables concurrently (such as on invocation of :c:func:`!free_pgtables`). +* Page tables may be *walked* under RCU alone, as page tables are freed only + after an RCU grace period has elapsed. However, any entry found must be + revalidated after the page table lock is taken (such as the + :c:func:`!pmd_same` recheck performed by :c:func:`!pte_offset_map_lock`) + before it is acted upon. Changing an entry requires the page table lock + and one of the locks that excludes teardown (any one of the mmap, VMA or + rmap locks). +* When traversing page tables under RCU alone it is important to take care + when operating upon leaf entries - if the value is operated upon (for + instance getting the folio associated with a PTE) an appropriate lock must + be taken to prevent concurrent modification. * Reads from and writes to page table entries must be *appropriately* atomic. See the section on atomicity below for details. * Populating previously empty entries requires that the mmap or VMA locks are diff --git a/Documentation/mm/vmemmap_dedup.rst b/Documentation/mm/vmemmap_dedup.rst index 9fa8642ded48..8c287ae3f86c 100644 --- a/Documentation/mm/vmemmap_dedup.rst +++ b/Documentation/mm/vmemmap_dedup.rst @@ -1,4 +1,3 @@ - .. SPDX-License-Identifier: GPL-2.0 ========================================= @@ -192,32 +191,7 @@ to 4 on HugeTLB pages. There's no remapping of vmemmap given that device-dax memory is not part of System RAM ranges initialized at boot. Thus the tail page deduplication -happens at a later stage when we populate the sections. HugeTLB reuses the -the head vmemmap page representing, whereas device-dax reuses the tail -vmemmap page. This results in only half of the savings compared to HugeTLB. +happens at a later stage when we populate the sections. -Deduplicated tail pages are not mapped read-only. - -Here's how things look like on device-dax after the sections are populated:: - - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PMD | +-----------+ | | | - | level | | 5 | ----------------------+ | | - | mapping | +-----------+ | | - | | | 6 | ------------------------+ | - | | +-----------+ | - | | | 7 | --------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ +Deduplicated tail pages are not mapped read-only. The mapping layout is the same +as HugeTLB. diff --git a/MAINTAINERS b/MAINTAINERS index 7336600952c8..bc961fb17b5a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12168,6 +12168,7 @@ F: Documentation/mm/hugetlbfs_reserv.rst F: Documentation/mm/vmemmap_dedup.rst F: fs/hugetlbfs/ F: include/linux/hugetlb.h +F: include/linux/vmemmap-optimization.h F: include/trace/events/hugetlbfs.h F: mm/hugetlb.c F: mm/hugetlb_cgroup.c @@ -17264,8 +17265,9 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: mm/gup.c F: mm/gup_test.c F: mm/gup_test.h +F: tools/mm/gup_bench.c +F: tools/testing/selftests/mm/gup.c F: tools/testing/selftests/mm/gup_longterm.c -F: tools/testing/selftests/mm/gup_test.c MEMORY MANAGEMENT - KSM (Kernel Samepage Merging) M: Andrew Morton @@ -17342,8 +17344,10 @@ F: Documentation/ABI/testing/sysfs-kernel-mm F: Documentation/ABI/testing/sysfs-kernel-mm-cma F: Documentation/ABI/testing/sysfs-kernel-mm-numa F: Documentation/admin-guide/mm/ +F: Documentation/core-api/memory-allocation.rst +F: Documentation/core-api/mm-api.rst F: Documentation/mm/ -F: drivers/char/mem.c +F: mm/char-mem.c F: include/linux/cma.h F: include/linux/dmapool.h F: include/linux/ioremap.h @@ -17361,6 +17365,7 @@ F: mm/mapping_dirty_helpers.c F: mm/page_idle.c F: mm/pgalloc-track.h F: mm/process_vm_access.c +F: tools/lib/mm/ F: tools/testing/selftests/mm/ MEMORY MANAGEMENT - NUMA MEMBLOCKS AND NUMA EMULATION @@ -17567,7 +17572,7 @@ L: linux-mm@kvack.org S: Maintained W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm -F: drivers/char/mem.c +F: mm/char-mem.c F: include/trace/events/mmap.h F: fs/proc/task_mmu.c F: fs/proc/task_nommu.c diff --git a/arch/Kconfig b/arch/Kconfig index c0a6a13b06a1..50601c0301e8 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -526,13 +526,6 @@ config HAVE_ARCH_JUMP_LABEL config HAVE_ARCH_JUMP_LABEL_RELATIVE bool -config MMU_GATHER_TABLE_FREE - bool - -config MMU_GATHER_RCU_TABLE_FREE - bool - select MMU_GATHER_TABLE_FREE - config MMU_GATHER_PAGE_SIZE bool @@ -548,7 +541,6 @@ config MMU_GATHER_MERGE_VMAS config MMU_GATHER_NO_GATHER bool - depends on MMU_GATHER_TABLE_FREE config ARCH_WANT_IRQS_OFF_ACTIVATE_MM bool diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 8fafbfabd1e5..19ed083d4f06 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -71,7 +71,6 @@ config ALPHA select ARCH_STACKWALK select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67 select MMU_GATHER_NO_RANGE - select MMU_GATHER_RCU_TABLE_FREE select SPARSE_IRQ select SPARSEMEM_EXTREME if SPARSEMEM select SYSCTL_EXCEPTION_TRACE diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 7cf685a694b6..c5966bd127f8 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -428,13 +428,6 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return pte; } -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern void paging_init(void); /* We have our own get_unmapped_area */ diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h index dfae070fe8d5..9b6c37f92e97 100644 --- a/arch/arc/include/asm/pgalloc.h +++ b/arch/arc/include/asm/pgalloc.h @@ -72,7 +72,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) set_p4d(p4dp, __p4d((unsigned long)pudp)); } -#define __pud_free_tlb(tlb, pmd, addr) pud_free((tlb)->mm, pmd) +#define __pud_free_tlb(tlb, pmd, addr) tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) #endif @@ -83,10 +83,10 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) set_pud(pudp, __pud((unsigned long)pmdp)); } -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +#define __pmd_free_tlb(tlb, pmd, addr) tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) #endif -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, addr) tlb_remove_ptdesc((tlb), page_ptdesc(pte)) #endif /* _ASM_ARC_PGALLOC_H */ diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h index c8f9273372c0..167b82fcfafe 100644 --- a/arch/arc/include/asm/pgtable-levels.h +++ b/arch/arc/include/asm/pgtable-levels.h @@ -98,8 +98,6 @@ /* * 1st level paging: pgd */ -#define pgd_ERROR(e) \ - pr_crit("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) #if CONFIG_PGTABLE_LEVELS > 3 @@ -115,9 +113,6 @@ /* * 2nd level paging: pud */ -#define pud_ERROR(e) \ - pr_crit("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) - #endif #if CONFIG_PGTABLE_LEVELS > 2 @@ -137,9 +132,6 @@ /* * 3rd level paging: pmd */ -#define pmd_ERROR(e) \ - pr_crit("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) - #define pmd_pfn(pmd) ((pmd_val(pmd) & PMD_MASK) >> PAGE_SHIFT) #define pfn_pmd(pfn,prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) @@ -165,9 +157,6 @@ /* * 4th level paging: pte */ -#define pte_ERROR(e) \ - pr_crit("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) - #define PFN_PTE_SHIFT PAGE_SHIFT #define pte_none(x) (!pte_val(x)) #define pte_present(x) (pte_val(x) & _PAGE_PRESENT) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index ffbc7f386131..0cc289a7184a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -42,7 +42,6 @@ config ARM select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_RT select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF @@ -135,7 +134,6 @@ config ARM select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ select HAVE_RUST if CPU_LITTLE_ENDIAN && CPU_32v7 && !KASAN diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 982795cf4563..8dd17d20faa3 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -44,13 +44,6 @@ #define LIBRARY_TEXT_START 0x0c000000 #ifndef __ASSEMBLY__ -extern void __pte_error(const char *file, int line, pte_t); -extern void __pmd_error(const char *file, int line, pmd_t); -extern void __pgd_error(const char *file, int line, pgd_t); - -#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte) -#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd) -#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd) /* * This is the lowest virtual address we can permit any user space diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index afbd2ebe5c39..ad04c806cc9d 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -753,23 +753,6 @@ void __readwrite_bug(const char *fn) } EXPORT_SYMBOL(__readwrite_bug); -#ifdef CONFIG_MMU -void __pte_error(const char *file, int line, pte_t pte) -{ - pr_err("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte)); -} - -void __pmd_error(const char *file, int line, pmd_t pmd) -{ - pr_err("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd)); -} - -void __pgd_error(const char *file, int line, pgd_t pgd) -{ - pr_err("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd)); -} -#endif - asmlinkage void __div0(void) { pr_err("Division by zero in kernel.\n"); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index d5ccf1d040c6..292392ded42f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -81,7 +81,6 @@ config ARM64 select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_RMM select ARCH_SUPPORTS_RT @@ -223,7 +222,6 @@ config ARM64 select HAVE_RELIABLE_STACKTRACE select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_FUNCTION_ARG_ACCESS_API - select MMU_GATHER_RCU_TABLE_FREE select HAVE_RSEQ select HAVE_RUST select HAVE_STACKPROTECTOR diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 7f7b97e09996..83f3b05fc784 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -23,9 +23,7 @@ unsigned long mte_copy_tags_from_user(void *to, const void __user *from, unsigned long n); unsigned long mte_copy_tags_to_user(void __user *to, void *from, unsigned long n); -int mte_save_tags(struct page *page); void mte_save_page_tags(const void *page_addr, void *tag_storage); -void mte_restore_tags(swp_entry_t entry, struct page *page); void mte_restore_page_tags(void *page_addr, const void *tag_storage); void mte_invalidate_tags(int type, pgoff_t offset); void mte_invalidate_tags_area(int type); diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 6000905a2e86..e89ec5f4787b 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -107,9 +107,6 @@ static inline void arch_leave_lazy_mmu_mode(void) __flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2, \ TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE) -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) - #ifdef CONFIG_ARM64_PA_BITS_52 static inline phys_addr_t __pte_to_phys(pte_t pte) { @@ -866,9 +863,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #if CONFIG_PGTABLE_LEVELS > 2 -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) - #define pud_none(pud) (!pud_val(pud)) #define pud_bad(pud) ((pud_val(pud) & PUD_TYPE_MASK) != \ PUD_TYPE_TABLE) @@ -960,9 +954,6 @@ static inline bool mm_pud_folded(const struct mm_struct *mm) } #define mm_pud_folded mm_pud_folded -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e)) - #define p4d_none(p4d) (pgtable_l4_enabled() && !p4d_val(p4d)) #define p4d_bad(p4d) (pgtable_l4_enabled() && \ ((p4d_val(p4d) & P4D_TYPE_MASK) != \ @@ -1088,9 +1079,6 @@ static inline bool mm_p4d_folded(const struct mm_struct *mm) } #define mm_p4d_folded mm_p4d_folded -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %016llx.\n", __FILE__, __LINE__, p4d_val(e)) - #define pgd_none(pgd) (pgtable_l5_enabled() && !pgd_val(pgd)) #define pgd_bad(pgd) (pgtable_l5_enabled() && \ ((pgd_val(pgd) & PGD_TYPE_MASK) != \ @@ -1217,9 +1205,6 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) #endif /* CONFIG_PGTABLE_LEVELS > 4 */ -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) - #define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) #define pgd_clear_fixmap() clear_fixmap(FIX_PGD) diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h index 90f61b17275e..0091ba12200e 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -11,9 +11,8 @@ bool can_set_direct_map(void); int set_memory_valid(unsigned long addr, int numpages, int enable); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages); +int set_direct_map_default_noflush(struct page *page, unsigned int numpages); bool kernel_page_present(struct page *page); int set_memory_encrypted(unsigned long addr, int numpages); diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c index 63e8d72f202a..f64202b69309 100644 --- a/arch/arm64/mm/mteswap.c +++ b/arch/arm64/mm/mteswap.c @@ -20,22 +20,17 @@ void mte_free_tag_storage(char *storage) kfree(storage); } -int mte_save_tags(struct page *page) +static int mte_save_tags(swp_entry_t entry, struct page *page) { void *tag_storage, *ret; - if (!page_mte_tagged(page)) - return 0; - tag_storage = mte_allocate_tag_storage(); if (!tag_storage) return -ENOMEM; mte_save_page_tags(page_address(page), tag_storage); - /* lookup the swap entry.val from the page */ - ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage, - GFP_KERNEL); + ret = xa_store(&mte_pages, entry.val, tag_storage, GFP_KERNEL); if (WARN(xa_is_err(ret), "Failed to store MTE tags")) { mte_free_tag_storage(tag_storage); return xa_err(ret); @@ -47,7 +42,7 @@ int mte_save_tags(struct page *page) return 0; } -void mte_restore_tags(swp_entry_t entry, struct page *page) +static void mte_restore_tags(swp_entry_t entry, struct page *page) { void *tags = xa_load(&mte_pages, entry.val); @@ -68,13 +63,6 @@ void mte_invalidate_tags(int type, pgoff_t offset) mte_free_tag_storage(tags); } -static inline void __mte_invalidate_tags(struct page *page) -{ - swp_entry_t entry = page_swap_entry(page); - - mte_invalidate_tags(swp_type(entry), swp_offset(entry)); -} - void mte_invalidate_tags_area(int type) { swp_entry_t entry = swp_entry(type, 0); @@ -102,15 +90,23 @@ int arch_prepare_to_swap(struct folio *folio) nr = folio_nr_pages(folio); for (i = 0; i < nr; i++) { - err = mte_save_tags(folio_page(folio, i)); + struct page *page = folio_page(folio, i); + + if (!page_mte_tagged(page)) + continue; + + err = mte_save_tags(folio_swap_entry(folio, i), page); if (err) goto out; } return 0; out: - while (i--) - __mte_invalidate_tags(folio_page(folio, i)); + while (i--) { + swp_entry_t swap = folio_swap_entry(folio, i); + + mte_invalidate_tags(swp_type(swap), swp_offset(swap)); + } return err; } diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 14b2a3801f40..de7bf95003ea 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -251,7 +251,7 @@ int set_memory_valid(unsigned long addr, int numpages, int enable) __pgprot(PTE_PRESENT_VALID_KERNEL)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages) { pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL); pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID); @@ -260,10 +260,10 @@ int set_direct_map_invalid_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int numpages) { pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE); pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY); @@ -272,7 +272,7 @@ int set_direct_map_default_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } int __set_memory_enc_dec(unsigned long addr, int numpages, bool encrypt) @@ -319,23 +319,7 @@ int __set_memory_enc_dec(unsigned long addr, int numpages, bool encrypt) __pgprot(PTE_PRESENT_INVALID)); } -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - unsigned long addr = (unsigned long)page_address(page); - - if (!can_set_direct_map()) - return 0; - - return set_memory_valid(addr, nr, valid); -} - #ifdef CONFIG_DEBUG_PAGEALLOC -/* - * This is - apart from the return value - doing the same - * thing as the new set_direct_map_valid_noflush() function. - * - * Unify? Explain the conceptual differences? - */ void __kernel_map_pages(struct page *page, int numpages, int enable) { if (!can_set_direct_map()) diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h index bafcd5823531..5ca77ff89ef9 100644 --- a/arch/csky/include/asm/pgtable.h +++ b/arch/csky/include/asm/pgtable.h @@ -23,10 +23,6 @@ #define PTRS_PER_PMD 1 #define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, (e).pte_low) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) #define PFN_PTE_SHIFT PAGE_SHIFT #define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT) diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h index 27b269e2870d..2fdb27afe703 100644 --- a/arch/hexagon/include/asm/pgtable.h +++ b/arch/hexagon/include/asm/pgtable.h @@ -94,9 +94,6 @@ #endif /* Any bigger and the PTE disappears. */ -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__,\ - pgd_val(e)) /* * Page Protection Constants. Includes (in this variant) cache attributes. diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index c346a9229542..0aad0004b010 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -69,7 +69,6 @@ config LOONGARCH select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS select ARCH_HAS_PTE_PROTNONE if 64BIT select ARCH_SUPPORTS_NUMA_BALANCING if NUMA - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SCHED_SMT if SMP select ARCH_SUPPORTS_SCHED_MC if SMP @@ -188,7 +187,6 @@ config LOONGARCH select IRQ_LOONGARCH_CPU select LOCK_MM_AND_FIND_VMA select MMU_GATHER_MERGE_VMAS if MMU - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA if MODULES select NEED_PER_CPU_EMBED_FIRST_CHUNK select NEED_PER_CPU_PAGE_FIRST_CHUNK diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index a05f6a4928dc..f87603135131 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -72,6 +72,7 @@ #include #include +#include #include #include @@ -135,19 +136,6 @@ struct vm_area_struct; #define ptep_get(ptep) READ_ONCE(*(ptep)) #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) -#ifndef __PAGETABLE_PMD_FOLDED -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#endif -#ifndef __PAGETABLE_PUD_FOLDED -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) -#endif -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern pte_t invalid_pte_table[PTRS_PER_PTE]; #ifndef __PAGETABLE_PUD_FOLDED diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 55dfaefd02c8..4bb01172fbc2 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -15,8 +15,7 @@ int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); bool kernel_page_present(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); #endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c index 4b46c5d30708..f801f7097f03 100644 --- a/arch/loongarch/mm/init.c +++ b/arch/loongarch/mm/init.c @@ -197,13 +197,15 @@ void __init __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t flags) { unsigned long addr = __fix_to_virt(idx); + char str[PTVAL_STR_MAX]; pte_t *ptep; BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); ptep = populate_kernel_pte(addr); if (!pte_none(ptep_get(ptep))) { - pte_ERROR(*ptep); + ptval_to_str(str, pte_val(*ptep)); + pr_err("unexpected set PTE at %lx in %s: %s\n", addr, __func__, str); return; } diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index 614ccc7afccb..a7dcff40f759 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -198,41 +198,22 @@ bool kernel_page_present(struct page *page) return pte_present(ptep_get(pte)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0)); + return __set_memory(addr, nr, PAGE_KERNEL, __pgprot(0)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); -} - -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - unsigned long addr = (unsigned long)page_address(page); - pgprot_t set, clear; - - if (addr < vm_map_base) - return 0; - - if (valid) { - set = PAGE_KERNEL; - clear = __pgprot(0); - } else { - set = __pgprot(0); - clear = __pgprot(_PAGE_PRESENT | _PAGE_VALID); - } - - return __set_memory(addr, nr, set, clear); + return __set_memory(addr, nr, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); } diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 11835eb59d94..eb84c3af92c0 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -29,6 +29,7 @@ config M68K select HAVE_ARCH_LIBGCC_H select HAVE_ARCH_SECCOMP select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TLB_REMOVE_TABLE if MMU_MOTOROLA select HAVE_ASM_MODVERSIONS select HAVE_DEBUG_BUGVERBOSE select HAVE_EFFICIENT_UNALIGNED_ACCESS if !CPU_HAS_NO_UNALIGNED diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h index fc5454d37da3..b53ff0950db2 100644 --- a/arch/m68k/include/asm/mcf_pgalloc.h +++ b/arch/m68k/include/asm/mcf_pgalloc.h @@ -39,10 +39,7 @@ extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, unsigned long address) { - struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); - - pagetable_dtor(ptdesc); - pagetable_free(ptdesc); + tlb_remove_ptdesc(tlb, virt_to_ptdesc(pgtable)); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm) diff --git a/arch/m68k/include/asm/mcf_pgtable.h b/arch/m68k/include/asm/mcf_pgtable.h index 189bb7b1e663..f45a882238db 100644 --- a/arch/m68k/include/asm/mcf_pgtable.h +++ b/arch/m68k/include/asm/mcf_pgtable.h @@ -137,12 +137,6 @@ static inline int pmd_bad2(pmd_t *pmd) { return 0; } #define pmd_present(pmd) (!pmd_none2(&(pmd))) static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; } -#define pte_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pte %08lx.\n", \ - __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", \ - __FILE__, __LINE__, pgd_val(e)) /* * The following only work if pte_present() is true. diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h index 1091fb0affbe..dcde40e8b5c6 100644 --- a/arch/m68k/include/asm/motorola_pgalloc.h +++ b/arch/m68k/include/asm/motorola_pgalloc.h @@ -17,6 +17,7 @@ enum m68k_table_types { extern void init_pointer_table(void *table, int type); extern void *get_pointer_table(struct mm_struct *mm, int type); extern int free_pointer_table(void *table, int type); +extern void __tlb_remove_table(void *table); /* * Allocate and free page tables. The xxx_kernel() versions are @@ -47,7 +48,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, unsigned long address) { - free_pointer_table(pgtable, TABLE_PTE); + tlb_remove_table(tlb, (void *)((unsigned long)pgtable | TABLE_PTE)); } @@ -61,10 +62,10 @@ static inline int pmd_free(struct mm_struct *mm, pmd_t *pmd) return free_pointer_table(pmd, TABLE_PMD); } -static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, - unsigned long address) +static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long address) { - return free_pointer_table(pmd, TABLE_PMD); + tlb_remove_table(tlb, (void *)((unsigned long)pmd | TABLE_PMD)); } diff --git a/arch/m68k/include/asm/motorola_pgtable.h b/arch/m68k/include/asm/motorola_pgtable.h index dcf6829b3eab..d9393b310add 100644 --- a/arch/m68k/include/asm/motorola_pgtable.h +++ b/arch/m68k/include/asm/motorola_pgtable.h @@ -131,14 +131,6 @@ static inline void pud_set(pud_t *pudp, pmd_t *pmdp) #define pud_clear(pudp) ({ pud_val(*pudp) = 0; }) #define pud_page(pud) (mem_map + ((unsigned long)(__va(pud_val(pud)) - PAGE_OFFSET) >> PAGE_SHIFT)) -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - - /* * The following only work if pte_present() is true. * Undefined behaviour if not.. diff --git a/arch/m68k/include/asm/sun3_pgtable.h b/arch/m68k/include/asm/sun3_pgtable.h index 80ca185a18a1..704442a391fd 100644 --- a/arch/m68k/include/asm/sun3_pgtable.h +++ b/arch/m68k/include/asm/sun3_pgtable.h @@ -119,13 +119,6 @@ static inline int pmd_present2 (pmd_t *pmd) { return pmd_val (*pmd) & SUN3_PMD_V #define pmd_present(pmd) (!pmd_none2(&(pmd))) static inline void pmd_clear (pmd_t *pmdp) { pmd_val (*pmdp) = 0; } - -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - - /* * The following only work if pte_present() is true. * Undefined behaviour if not... diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index b30aa69a73a6..f3efa0d13963 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -103,6 +104,8 @@ static struct list_head ptable_list[3] = { LIST_HEAD_INIT(ptable_list[2]), }; +static DEFINE_SPINLOCK(ptable_lock); + #define PD_PTABLE(ptdesc) ((ptable_desc *)&(virt_to_ptdesc((void *)(ptdesc))->pt_list)) #define PD_PTDESC(ptable) (list_entry(ptable, struct ptdesc, pt_list)) #define PD_MARKBITS(dp) (*(unsigned int *)&PD_PTDESC(dp)->pt_index) @@ -139,52 +142,65 @@ void __init init_pointer_table(void *table, int type) return; } +/* + * For a pointer table for a user process address space, a + * table is taken from a ptdesc allocated for the purpose. Each + * ptdesc can hold 8 pointer tables. The ptdesc is remapped in + * virtual address space to be noncacheable. + */ +static void *add_pointer_table(struct mm_struct *mm, int type) +{ + struct ptdesc *ptdesc; + ptable_desc *new; + void *pt_addr; + + ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); + if (!ptdesc) + return NULL; + + pt_addr = ptdesc_address(ptdesc); + + switch (type) { + case TABLE_PTE: + /* + * m68k doesn't have SPLIT_PTE_PTLOCKS for not having + * SMP. + */ + pagetable_pte_ctor(mm, ptdesc); + break; + case TABLE_PMD: + pagetable_pmd_ctor(mm, ptdesc); + break; + case TABLE_PGD: + pagetable_pgd_ctor(ptdesc); + break; + } + + mmu_page_ctor(pt_addr); + + new = PD_PTABLE(pt_addr); + + PD_MARKBITS(new) = ptable_mask(type) - 1; + scoped_guard(spinlock_bh, &ptable_lock) + list_add(new, &ptable_list[type]); + + return (pmd_t *)pt_addr; +} + void *get_pointer_table(struct mm_struct *mm, int type) { - ptable_desc *dp = ptable_list[type].next; - unsigned int mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp); unsigned int tmp, off; + unsigned long mask; + ptable_desc *dp; + void *ret; + + spin_lock_bh(&ptable_lock); + dp = ptable_list[type].next; + mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp); - /* - * For a pointer table for a user process address space, a - * table is taken from a ptdesc allocated for the purpose. Each - * ptdesc can hold 8 pointer tables. The ptdesc is remapped in - * virtual address space to be noncacheable. - */ if (mask == 0) { - struct ptdesc *ptdesc; - ptable_desc *new; - void *pt_addr; - - ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); - if (!ptdesc) - return NULL; - - pt_addr = ptdesc_address(ptdesc); - - switch (type) { - case TABLE_PTE: - /* - * m68k doesn't have SPLIT_PTE_PTLOCKS for not having - * SMP. - */ - pagetable_pte_ctor(mm, ptdesc); - break; - case TABLE_PMD: - pagetable_pmd_ctor(mm, ptdesc); - break; - case TABLE_PGD: - pagetable_pgd_ctor(ptdesc); - break; - } - - mmu_page_ctor(pt_addr); - - new = PD_PTABLE(pt_addr); - PD_MARKBITS(new) = ptable_mask(type) - 1; - list_add_tail(new, dp); - - return (pmd_t *)pt_addr; + spin_unlock_bh(&ptable_lock); + return add_pointer_table(mm, type); } for (tmp = 1, off = 0; (mask & tmp) == 0; tmp <<= 1, off += ptable_size(type)) @@ -194,7 +210,10 @@ void *get_pointer_table(struct mm_struct *mm, int type) /* move to end of list */ list_move_tail(dp, &ptable_list[type]); } - return ptdesc_address(PD_PTDESC(dp)) + off; + + ret = ptdesc_address(PD_PTDESC(dp)) + off; + spin_unlock_bh(&ptable_lock); + return ret; } int free_pointer_table(void *table, int type) @@ -204,6 +223,8 @@ int free_pointer_table(void *table, int type) unsigned long pt_addr = ptable & PAGE_MASK; unsigned int mask = 1U << ((ptable - pt_addr)/ptable_size(type)); + spin_lock_bh(&ptable_lock); + dp = PD_PTABLE(pt_addr); if (PD_MARKBITS (dp) & mask) panic ("table already free!"); @@ -213,6 +234,8 @@ int free_pointer_table(void *table, int type) if (PD_MARKBITS(dp) == ptable_mask(type)) { /* all tables in ptdesc are free, free ptdesc */ list_del(dp); + spin_unlock_bh(&ptable_lock); + mmu_page_dtor((void *)pt_addr); pagetable_dtor_free(virt_to_ptdesc((void *)pt_addr)); return 1; @@ -223,9 +246,21 @@ int free_pointer_table(void *table, int type) */ list_move(dp, &ptable_list[type]); } + + spin_unlock_bh(&ptable_lock); return 0; } +void __tlb_remove_table(void *table) +{ + /* The bottom 2 bits are used to encode page table type. */ + const unsigned long encoded = (unsigned long)table; + void *addr = (void *)(encoded & ~3UL); + const int type = encoded & 3; + + free_pointer_table(addr, type); +} + /* size of memory already mapped in head.S */ extern __initdata unsigned long m68k_init_mapped_size; diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h index 084a8a0dc239..ffee6a009219 100644 --- a/arch/microblaze/include/asm/pgalloc.h +++ b/arch/microblaze/include/asm/pgalloc.h @@ -25,7 +25,7 @@ extern void __bad_pte(pmd_t *pmd); extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm); -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, (pte)) +#define __pte_free_tlb(tlb, pte, addr) tlb_remove_ptdesc((tlb), page_ptdesc(pte)) #define pmd_populate(mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)page_address(pte)) diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index 7678c040a2fd..72708f9af1c0 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -103,13 +103,6 @@ extern pte_t *va_to_pte(unsigned long address); #define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT) #define KERNEL_PGD_PTRS (PTRS_PER_PGD-USER_PGD_PTRS) -#define pte_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pte "PTE_FMT".\n", \ - __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", \ - __FILE__, __LINE__, pgd_val(e)) - /* * Bits in a linux-style PTE. These match the bits in the * (hardware-defined) PTE as closely as possible. diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index d7c67cebe065..ed17c9b1624c 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -97,7 +97,6 @@ config MIPS select IRQ_FORCED_THREADING select ISA if EISA select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_REL if MODULES select MODULES_USE_ELF_RELA if MODULES && 64BIT select PERF_USE_VMALLOC diff --git a/arch/mips/include/asm/pgtable-32.h b/arch/mips/include/asm/pgtable-32.h index 92b7591aac2a..ef1001ab09c5 100644 --- a/arch/mips/include/asm/pgtable-32.h +++ b/arch/mips/include/asm/pgtable-32.h @@ -104,16 +104,6 @@ extern int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, # define VMALLOC_END (FIXADDR_START-2*PAGE_SIZE) #endif -#ifdef CONFIG_PHYS_ADDR_T_64BIT -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %016Lx.\n", __FILE__, __LINE__, pte_val(e)) -#else -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#endif -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern void load_pgd(unsigned long pg_dir); extern pte_t invalid_pte_table[PTRS_PER_PTE]; diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h index 6e854bb11f37..785fc37bab94 100644 --- a/arch/mips/include/asm/pgtable-64.h +++ b/arch/mips/include/asm/pgtable-64.h @@ -151,19 +151,6 @@ #define MODULES_END (FIXADDR_START-2*PAGE_SIZE) #endif -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) -#ifndef __PAGETABLE_PMD_FOLDED -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#endif -#ifndef __PAGETABLE_PUD_FOLDED -#define pud_ERROR(e) \ - printk("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) -#endif -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern pte_t invalid_pte_table[PTRS_PER_PTE]; #ifndef __PAGETABLE_PUD_FOLDED diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h index d389aa9ca57c..272707d48f1b 100644 --- a/arch/nios2/include/asm/pgtable.h +++ b/arch/nios2/include/asm/pgtable.h @@ -223,13 +223,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) return pmd_val(pmd); } -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", \ - __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", \ - __FILE__, __LINE__, pgd_val(e)) - /* * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that * are !pte_none() && !pte_present(). diff --git a/arch/openrisc/include/asm/pgtable.h b/arch/openrisc/include/asm/pgtable.h index 6b89996d0b62..13afcc0bd863 100644 --- a/arch/openrisc/include/asm/pgtable.h +++ b/arch/openrisc/include/asm/pgtable.h @@ -338,13 +338,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #define pte_pfn(x) ((unsigned long)(((x).pte)) >> PAGE_SHIFT) #define pfn_pte(pfn, prot) __pte((((pfn) << PAGE_SHIFT)) | pgprot_val(prot)) -#define pte_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pte %p(%08lx).\n", \ - __FILE__, __LINE__, &(e), pte_val(e)) -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %p(%08lx).\n", \ - __FILE__, __LINE__, &(e), pgd_val(e)) - extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; /* defined in head.S */ struct vm_area_struct; diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index d3afac2f0d9b..77f67028ad89 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -80,7 +80,6 @@ config PARISC select GENERIC_CLOCKEVENTS select CPU_NO_EFFICIENT_FFS select THREAD_INFO_IN_TASK - select MMU_GATHER_RCU_TABLE_FREE select NEED_DMA_MAP_STATE select NEED_SG_DMA_LENGTH select HAVE_ARCH_KGDB diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 467b8547ac8b..f6899375cb43 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -75,15 +75,6 @@ extern void __update_cache(pte_t pte); #endif /* !__ASSEMBLER__ */ -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#if CONFIG_PGTABLE_LEVELS == 3 -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, (unsigned long)pmd_val(e)) -#endif -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, (unsigned long)pgd_val(e)) - /* This is the size of the initially mapped kernel memory */ #if defined(CONFIG_64BIT) || defined(CONFIG_KALLSYMS) #define KERNEL_INITIAL_ORDER 26 /* 1<<26 = 64MB */ diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index bf9f192c826e..84e7309a8266 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -160,8 +160,7 @@ static inline void unmap_uncached_pte(pmd_t * pmd, unsigned long vaddr, if (pmd_none(*pmd)) return; if (pmd_bad(*pmd)) { - pmd_ERROR(*pmd); - pmd_clear(pmd); + pmd_clear_bad(pmd); return; } pte = pte_offset_kernel(pmd, vaddr); @@ -196,8 +195,7 @@ static inline void unmap_uncached_pmd(pgd_t * dir, unsigned long vaddr, if (pgd_none(*dir)) return; if (pgd_bad(*dir)) { - pgd_ERROR(*dir); - pgd_clear(dir); + pgd_clear_bad(dir); return; } pmd = pmd_offset(pud_offset(p4d_offset(dir, vaddr), vaddr), vaddr); diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 7f7910f38a67..1b5ab1963be2 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -305,7 +305,6 @@ config PPC select KASAN_VMALLOC if KASAN && EXECMEM select LOCK_MM_AND_FIND_VMA select MMU_GATHER_PAGE_SIZE - select MMU_GATHER_RCU_TABLE_FREE select HAVE_ARCH_TLB_REMOVE_TABLE select MMU_GATHER_MERGE_VMAS select MMU_LAZY_TLB_SHOOTDOWN if PPC_BOOK3S_64 diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index e18a4fa282a1..835e84caee13 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -203,8 +203,6 @@ void unmap_kernel_page(unsigned long va); /* Bits to mask out from a PGD to get to the PUD page */ #define PGD_MASKED_BITS 0 -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) /* * Bits in a linux-style PTE. These match the bits in the * (hardware-defined) PowerPC PTE as closely as possible. diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index f4db7d7fbd5c..dff8790a047d 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -991,13 +991,6 @@ static inline pmd_t *pud_pgtable(pud_t pud) return (pmd_t *)__va(pud_val(pud) & ~PUD_MASKED_BITS); } -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) { if (radix_enabled()) { diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 496ecc65ac25..f17afde89fa1 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -51,8 +51,6 @@ #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08llx.\n", __FILE__, __LINE__, (unsigned long long)pgd_val(e)) /* * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary diff --git a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h index fb6fa1d4e074..75cf3c331b92 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h @@ -82,9 +82,6 @@ extern struct page *p4d_page(p4d_t p4d); #endif /* !__ASSEMBLER__ */ -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) - /* * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() */ #define remap_4k_pfn(vma, addr, pfn, prot) \ diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 661eb3820d12..446dde8b6ead 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -159,11 +159,6 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __young; \ }) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - /* * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that * are !pte_none() && !pte_present(). diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index cf692b2b5f7b..a3332f32ffb2 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -1041,13 +1042,17 @@ static pte_t * __meminit radix__vmemmap_pte_populate(pmd_t *pmdp, unsigned long /* * When a PTE/PMD entry is freed from the init_mm * there's a free_pages() call to this page allocated - * above. Thus this get_page() is paired with the + * above. Thus this try_get_page() is paired with the * put_page_testzero() on the freeing path. * This can only called by certain ZONE_DEVICE path, * and through vmemmap_populate_compound_pages() when * slab is available. + * + * Use try_get_page() to prevent the shared page refcount + * from overflowing. */ - get_page(reuse); + if (!try_get_page(reuse)) + return NULL; p = page_to_virt(reuse); pr_debug("Tail page reuse vmemmap mapping\n"); } @@ -1217,92 +1222,6 @@ base_mapping: return 0; } -static pte_t * __meminit radix__vmemmap_populate_address(unsigned long addr, int node, - struct vmem_altmap *altmap, - struct page *reuse) -{ - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - - pgd = pgd_offset_k(addr); - p4d = p4d_offset(pgd, addr); - pud = vmemmap_pud_alloc(p4d, node, addr); - if (!pud) - return NULL; - pmd = vmemmap_pmd_alloc(pud, node, addr); - if (!pmd) - return NULL; - if (pmd_leaf(*pmd)) - /* - * The second page is mapped as a hugepage due to a nearby request. - * Force our mapping to page size without deduplication - */ - return NULL; - pte = vmemmap_pte_alloc(pmd, node, addr); - if (!pte) - return NULL; - radix__vmemmap_pte_populate(pmd, addr, node, NULL, NULL); - vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - - return pte; -} - -static pte_t * __meminit vmemmap_compound_tail_page(unsigned long addr, - unsigned long pfn_offset, int node) -{ - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - unsigned long map_addr; - - /* the second vmemmap page which we use for duplication */ - map_addr = addr - pfn_offset * sizeof(struct page) + PAGE_SIZE; - pgd = pgd_offset_k(map_addr); - p4d = p4d_offset(pgd, map_addr); - pud = vmemmap_pud_alloc(p4d, node, map_addr); - if (!pud) - return NULL; - pmd = vmemmap_pmd_alloc(pud, node, map_addr); - if (!pmd) - return NULL; - if (pmd_leaf(*pmd)) - /* - * The second page is mapped as a hugepage due to a nearby request. - * Force our mapping to page size without deduplication - */ - return NULL; - pte = vmemmap_pte_alloc(pmd, node, map_addr); - if (!pte) - return NULL; - /* - * Check if there exist a mapping to the left - */ - if (pte_none(*pte)) { - /* - * Populate the head page vmemmap page. - * It can fall in different pmd, hence - * vmemmap_populate_address() - */ - pte = radix__vmemmap_populate_address(map_addr - PAGE_SIZE, node, NULL, NULL); - if (!pte) - return NULL; - /* - * Populate the tail pages vmemmap page - */ - pte = radix__vmemmap_pte_populate(pmd, map_addr, node, NULL, NULL); - if (!pte) - return NULL; - vmemmap_verify(pte, node, map_addr, map_addr + PAGE_SIZE); - return pte; - } - return pte; -} - int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, unsigned long start, unsigned long end, int node, @@ -1320,6 +1239,12 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, pud_t *pud; pmd_t *pmd; pte_t *pte; + struct page *tail_page; + unsigned int order = pfn_to_section_compound_order(start_pfn); + + tail_page = vmemmap_shared_tail_page(order, device_zone(node)); + if (!tail_page) + return -ENOMEM; for (addr = start; addr < end; addr = next) { @@ -1343,16 +1268,15 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte_none(*pte)) { /* * This could be because we already have a compound - * page whose VMEMMAP_RESERVE_NR pages were mapped and + * page whose retained vmemmap page was mapped and * this request fall in those pages. */ next = addr + PAGE_SIZE; continue; } else { - unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); + unsigned long nr_pages = 1UL << order; unsigned long addr_pfn = page_to_pfn((struct page *)addr); unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages); - pte_t *tail_page_pte; /* * if the address is aligned to huge page size it is the @@ -1365,35 +1289,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - /* - * Populate the tail pages vmemmap page - * It can fall in different pmd, hence - * vmemmap_populate_address() - */ - pte = radix__vmemmap_populate_address(addr + PAGE_SIZE, node, NULL, NULL); - if (!pte) - return -ENOMEM; - - next = addr + 2 * PAGE_SIZE; - continue; - } - /* - * get the 2nd mapping details - * Also create it if that doesn't exist - */ - tail_page_pte = vmemmap_compound_tail_page(addr, pfn_offset, node); - if (!tail_page_pte) { - - pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, NULL); - if (!pte) - return -ENOMEM; - vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - next = addr + PAGE_SIZE; continue; } - pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, pte_page(*tail_page_pte)); + pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, tail_page); if (!pte) return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig index b5ad7c173ef0..dd8f6060fb7a 100644 --- a/arch/powerpc/platforms/powernv/Kconfig +++ b/arch/powerpc/platforms/powernv/Kconfig @@ -17,7 +17,6 @@ config PPC_POWERNV select PPC_DOORBELL select MMU_NOTIFIER select FORCE_SMP - select ARCH_SUPPORTS_PER_VMA_LOCK select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU default y diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index 74910ce3a541..7d125e288f6e 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -23,7 +23,6 @@ config PPC_PSERIES select HOTPLUG_CPU select FORCE_SMP select SWIOTLB - select ARCH_SUPPORTS_PER_VMA_LOCK select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU default y diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 780deafe6bda..f35b7b1584ec 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -72,7 +72,6 @@ config RISCV select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU - select ARCH_SUPPORTS_PER_VMA_LOCK if MMU select ARCH_HAS_PTE_PROTNONE if MMU select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK @@ -208,7 +207,6 @@ config RISCV select IRQ_FORCED_THREADING select KASAN_VMALLOC if KASAN select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE if SMP && MMU select MODULES_USE_ELF_RELA if MODULES select OF select OF_EARLY_FLATTREE diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 709a36fb4323..b4bbae55e931 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -76,12 +76,6 @@ typedef struct page *pgtable_t; #define __pgd(x) ((pgd_t) { (x) }) #define __pgprot(x) ((pgprot_t) { (x) }) -#ifdef CONFIG_64BIT -#define PTE_FMT "%016lx" -#else -#define PTE_FMT "%08lx" -#endif - #if defined(CONFIG_64BIT) && defined(CONFIG_MMU) /* * We override this value as its generic definition uses __pa too early in diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 6e789fa58514..ae23182b572c 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -264,15 +264,6 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) return __page_val_to_pfn(pmd_val(pmd)); } -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) - -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) - -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) - static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) { if (pgtable_l4_enabled) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 40b1ed4f3ea8..4c8fc6845503 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -556,10 +556,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) return __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val); } -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e)) - - /* Commit new configuration to MMU hardware */ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ef59e1716a2c..e9f9960c1947 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -40,9 +40,8 @@ static inline int set_kernel_memory(char *startp, char *endp, } #endif -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); bool kernel_page_present(struct page *page); #endif /* __ASSEMBLER__ */ diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index fb37b0b67efe..857f9a55039c 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 3f76db3d2769..5b3cf326455d 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -374,33 +374,18 @@ int set_memory_nx(unsigned long addr, int numpages) return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, __pgprot(0), __pgprot(_PAGE_PRESENT)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, PAGE_KERNEL, __pgprot(_PAGE_EXEC)); } -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - pgprot_t set, clear; - - if (valid) { - set = PAGE_KERNEL; - clear = __pgprot(_PAGE_EXEC); - } else { - set = __pgprot(0); - clear = __pgprot(_PAGE_PRESENT); - } - - return __set_memory((unsigned long)page_address(page), nr, set, clear); -} - #ifdef CONFIG_DEBUG_PAGEALLOC static int debug_pagealloc_set_page(pte_t *pte, unsigned long addr, void *data) { diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 10ada552d870..a8b891cda4a7 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -157,7 +157,6 @@ config S390 select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_USES_CFI_GENERIC_LLVM_PASS if CC_IS_CLANG select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF @@ -268,7 +267,6 @@ config S390 select LOCK_MM_AND_FIND_VMA select MMU_GATHER_MERGE_VMAS select MMU_GATHER_NO_GATHER - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE if PCI select NEED_PER_CPU_EMBED_FIRST_CHUNK diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index e882663a58e7..2d5c2ab06de9 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -68,17 +68,6 @@ extern unsigned long zero_page_mask; /* TODO: s390 cannot support io_remap_pfn_range... */ -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - /* * The vmalloc and module area will always be on the topmost area of the * kernel mapping. 512GB are reserved for vmalloc by default. diff --git a/arch/s390/include/asm/set_memory.h b/arch/s390/include/asm/set_memory.h index 94092f4ae764..e3562bf0c1aa 100644 --- a/arch/s390/include/asm/set_memory.h +++ b/arch/s390/include/asm/set_memory.h @@ -60,9 +60,8 @@ __SET_MEMORY_FUNC(set_memory_rox, SET_MEMORY_RO | SET_MEMORY_X) __SET_MEMORY_FUNC(set_memory_rwnx, SET_MEMORY_RW | SET_MEMORY_NX) __SET_MEMORY_FUNC(set_memory_4k, SET_MEMORY_4K) -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); bool kernel_page_present(struct page *page); #endif diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 1e202e3d08e7..80e834e8b8e1 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -382,26 +382,14 @@ out: return rc; } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_INV); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_INV); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_DEF); -} - -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - unsigned long flags; - - if (valid) - flags = SET_MEMORY_DEF; - else - flags = SET_MEMORY_INV; - - return __set_memory((unsigned long)page_to_virt(page), nr, flags); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_DEF); } bool kernel_page_present(struct page *page) diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index d60f1d5a94c0..fe859def918c 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -33,6 +33,7 @@ config SUPERH select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_KGDB select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TLB_REMOVE_TABLE if X2TLB select HAVE_ARCH_TRACEHOOK select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h index 6fe7123d38fa..67ce7fa23fa1 100644 --- a/arch/sh/include/asm/pgalloc.h +++ b/arch/sh/include/asm/pgalloc.h @@ -17,7 +17,11 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address); extern void pmd_free(struct mm_struct *mm, pmd_t *pmd); -#define __pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, (pmdp)) +extern void __tlb_remove_table(void *table); + +/* PMDs are slab-allocated, tag so they are freed correctly. */ +#define __pmd_free_tlb(tlb, pmdp, addr) \ + tlb_remove_table((tlb), (void *)((unsigned long)(pmdp) | 1)) #endif static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, diff --git a/arch/sh/include/asm/pgtable-3level.h b/arch/sh/include/asm/pgtable-3level.h index d1ce73f3bd85..3f4d747f30f4 100644 --- a/arch/sh/include/asm/pgtable-3level.h +++ b/arch/sh/include/asm/pgtable-3level.h @@ -25,9 +25,6 @@ #define PTRS_PER_PMD ((1 << PGDIR_SHIFT) / PMD_SIZE) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) - typedef union { struct { unsigned long pmd_low; diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h index 5f51af18997b..cde1bf0c6734 100644 --- a/arch/sh/include/asm/pgtable_32.h +++ b/arch/sh/include/asm/pgtable_32.h @@ -400,19 +400,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT) #define pmd_page(pmd) (virt_to_page(pmd_val(pmd))) -#ifdef CONFIG_X2TLB -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%08lx%08lx).\n", __FILE__, __LINE__, \ - &(e), (e).pte_high, (e).pte_low) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) -#else -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) -#endif - /* * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that * are !pte_none() && !pte_present(). diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 110308bdef01..93921109f4e6 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -52,26 +52,24 @@ static pte_t *__get_pte_phys(unsigned long addr) pmd_t *pmd; pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { - pgd_ERROR(*pgd); + if (pgd_none(*pgd)) return NULL; - } p4d = p4d_alloc(NULL, pgd, addr); if (unlikely(!p4d)) { - p4d_ERROR(*p4d); + pr_err("allocating p4d table failed\n"); return NULL; } pud = pud_alloc(NULL, p4d, addr); if (unlikely(!pud)) { - pud_ERROR(*pud); + pr_err("allocating pud table failed\n"); return NULL; } pmd = pmd_alloc(NULL, pud, addr); if (unlikely(!pmd)) { - pmd_ERROR(*pmd); + pr_err("allocating pmd table failed\n"); return NULL; } @@ -84,7 +82,11 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot) pte = __get_pte_phys(addr); if (!pte_none(*pte)) { - pte_ERROR(*pte); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pte_val(*pte)); + pr_err("unexpected set PTE at %lx in %s: bad pte %p(%s).\n", + addr, __func__, pte, str); return; } diff --git a/arch/sh/mm/pgtable.c b/arch/sh/mm/pgtable.c index 3a4085ea0161..f6184b86b89c 100644 --- a/arch/sh/mm/pgtable.c +++ b/arch/sh/mm/pgtable.c @@ -56,4 +56,24 @@ void pmd_free(struct mm_struct *mm, pmd_t *pmd) { kmem_cache_free(pmd_cachep, pmd); } + +static void __tlb_remove_table_slab(void *table) +{ + kmem_cache_free(pmd_cachep, table); +} + +static void __tlb_remove_table_pgtable(void *table) +{ + pagetable_dtor_free(table); +} + +void __tlb_remove_table(void *table) +{ + const unsigned long addr = (unsigned long)table; + + if (addr & 1) + __tlb_remove_table_slab((void *)(addr & ~1UL)); + else + __tlb_remove_table_pgtable(table); +} #endif /* PAGETABLE_LEVELS > 2 */ diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ab77d3f2536e..742ffff8c37f 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -64,6 +64,7 @@ config SPARC32 select HAVE_UID16 select HAVE_PAGE_SIZE_4KB select LOCK_MM_AND_FIND_VMA + select HAVE_ARCH_TLB_REMOVE_TABLE select OLD_SIGACTION select ZONE_DMA @@ -75,8 +76,7 @@ config SPARC64 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_KRETPROBES select HAVE_KPROBES - select MMU_GATHER_RCU_TABLE_FREE if SMP - select HAVE_ARCH_TLB_REMOVE_TABLE if SMP + select HAVE_ARCH_TLB_REMOVE_TABLE select MMU_GATHER_MERGE_VMAS select MMU_GATHER_NO_FLUSH_CACHE select HAVE_ARCH_TRANSPARENT_HUGEPAGE diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h index 4f73e87b22a3..36010852ba0c 100644 --- a/arch/sparc/include/asm/pgalloc_32.h +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -48,7 +48,9 @@ static inline void free_pmd_fast(pmd_t * pmd) } #define pmd_free(mm, pmd) free_pmd_fast(pmd) -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) + +#define __pmd_free_tlb(tlb, pmd, addr) \ + tlb_remove_table((tlb), (void *)((unsigned long)(pmd) | 1UL)) #define pmd_populate(mm, pmd, pte) pmd_set(pmd, pte) @@ -72,6 +74,7 @@ static inline void free_pte_fast(pte_t *pte) #define pte_free_kernel(mm, pte) free_pte_fast(pte) void pte_free(struct mm_struct * mm, pgtable_t pte); -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) +void __tlb_remove_table(void *table); +#define __pte_free_tlb(tlb, pte, addr) tlb_remove_table((tlb), (void *)(pte)) #endif /* _SPARC_PGALLOC_H */ diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index caa7632be4c2..b5055d259b74 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -74,8 +74,6 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable); void pgtable_free(void *table, bool is_page); -#ifdef CONFIG_SMP - struct mmu_gather; void tlb_remove_table(struct mmu_gather *, void *); @@ -96,12 +94,6 @@ static inline void __tlb_remove_table(void *_table) is_page = true; pgtable_free(table, is_page); } -#else /* CONFIG_SMP */ -static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) -{ - pgtable_free(table, is_page); -} -#endif /* !CONFIG_SMP */ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, unsigned long address) diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index f89b1250661d..5a5f54a090f5 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -40,9 +40,6 @@ void load_mmu(void); unsigned long calc_highpages(void); unsigned long __init bootmem_init(unsigned long *pages_avail); -#define pte_ERROR(e) __builtin_trap() -#define pmd_ERROR(e) __builtin_trap() -#define pgd_ERROR(e) __builtin_trap() #define PTRS_PER_PTE 64 #define PTRS_PER_PMD 64 diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 0837ebbc5dce..44d1333065a6 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -96,16 +96,6 @@ bool kern_addr_valid(unsigned long addr); #define PTRS_PER_PUD (1UL << PUD_BITS) #define PTRS_PER_PGD (1UL << PGDIR_BITS) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \ - __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n", \ - __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \ - __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0)) - #endif /* !(__ASSEMBLER__) */ /* PTE bits which are the same in SUN4U and SUN4V format. */ diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h index 3037187482db..f5f9631685d5 100644 --- a/arch/sparc/include/asm/tlb_64.h +++ b/arch/sparc/include/asm/tlb_64.h @@ -29,9 +29,7 @@ void flush_tlb_pending(void); * and therefore we don't need a TLBI when freeing page-table pages. */ -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE #define tlb_needs_table_invalidate() (false) -#endif #include diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c index 32a5c1d9459c..c309e27973ce 100644 --- a/arch/sparc/lib/bitext.c +++ b/arch/sparc/lib/bitext.c @@ -22,8 +22,6 @@ * @align: requested alignment * * Returns offset in the map or -1 if out of space. - * - * Not safe to call from an interrupt (uses spin_lock). */ int bit_map_string_get(struct bit_map *t, int len, int align) { @@ -31,6 +29,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) int off_new; int align1; int i, color; + unsigned long flags; if (t->num_colors) { /* align is overloaded to be the page color */ @@ -50,7 +49,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) BUG(); color &= align1; - spin_lock(&t->lock); + spin_lock_irqsave(&t->lock, flags); if (len < t->last_size) offset = t->first_free; else @@ -64,7 +63,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) if (offset >= t->size) offset = 0; if (count + len > t->size) { - spin_unlock(&t->lock); + spin_unlock_irqrestore(&t->lock, flags); /* P3 */ printk(KERN_ERR "bitmap out: size %d used %d off %d len %d align %d count %d\n", t->size, t->used, offset, len, align, count); @@ -90,7 +89,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) t->last_off = 0; t->used += len; t->last_size = len; - spin_unlock(&t->lock); + spin_unlock_irqrestore(&t->lock, flags); return offset; } } @@ -103,10 +102,11 @@ int bit_map_string_get(struct bit_map *t, int len, int align) void bit_map_clear(struct bit_map *t, int offset, int len) { int i; + unsigned long flags; if (t->used < len) BUG(); /* Much too late to do any good, but alas... */ - spin_lock(&t->lock); + spin_lock_irqsave(&t->lock, flags); for (i = 0; i < len; i++) { if (test_bit(offset + i, t->map) == 0) BUG(); @@ -115,7 +115,7 @@ void bit_map_clear(struct bit_map *t, int offset, int len) if (offset < t->first_free) t->first_free = offset; t->used -= len; - spin_unlock(&t->lock); + spin_unlock_irqrestore(&t->lock, flags); } void bit_map_init(struct bit_map *t, unsigned long *map, int size) diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index 9a74902ad181..1c277ab3cdb8 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -340,38 +340,60 @@ pgd_t *get_pgd_fast(void) * Alignments up to the page size are the same for physical and virtual * addresses of the nocache area. */ + +static DEFINE_SPINLOCK(pte_page_lock); + pgtable_t pte_alloc_one(struct mm_struct *mm) { + unsigned long flags; pte_t *ptep; struct page *page; if (!(ptep = pte_alloc_one_kernel(mm))) return NULL; page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); - spin_lock(&mm->page_table_lock); + spin_lock_irqsave(&pte_page_lock, flags); if (page_ref_inc_return(page) == 2 && !pagetable_pte_ctor(mm, page_ptdesc(page))) { page_ref_dec(page); ptep = NULL; } - spin_unlock(&mm->page_table_lock); + spin_unlock_irqrestore(&pte_page_lock, flags); return ptep; } -void pte_free(struct mm_struct *mm, pgtable_t ptep) +static void __pte_free(pgtable_t ptep) { struct page *page; + unsigned long flags; page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); - spin_lock(&mm->page_table_lock); + spin_lock_irqsave(&pte_page_lock, flags); if (page_ref_dec_return(page) == 1) pagetable_dtor(page_ptdesc(page)); - spin_unlock(&mm->page_table_lock); + spin_unlock_irqrestore(&pte_page_lock, flags); srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE); } +void pte_free(struct mm_struct *mm, pgtable_t ptep) +{ + __pte_free(ptep); +} + +void __tlb_remove_table(void *table) +{ + const unsigned long encoded = (unsigned long)table; + const unsigned long addr = encoded & ~1UL; + const bool is_pmd = encoded & 1; + + if (is_pmd) + free_pmd_fast((pmd_t *)addr); + else + __pte_free((pgtable_t)addr); +} + /* context handling - a dynamically sized pool is used */ #define NO_CONTEXT -1 diff --git a/arch/um/Kconfig b/arch/um/Kconfig index 59d62c74fc41..7245e3ada1c5 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -45,7 +45,6 @@ config UML select HAVE_SYSCALL_TRACEPOINTS select THREAD_INFO_IN_TASK select SPARSE_IRQ - select MMU_GATHER_RCU_TABLE_FREE if MMU select UACCESS_MEMCPY if !MMU select GENERIC_STRNLEN_USER if !MMU select GENERIC_STRNCPY_FROM_USER if !MMU diff --git a/arch/um/include/asm/pgtable-2level.h b/arch/um/include/asm/pgtable-2level.h index 14ec16f92ce4..fa625f5b5ef7 100644 --- a/arch/um/include/asm/pgtable-2level.h +++ b/arch/um/include/asm/pgtable-2level.h @@ -24,13 +24,6 @@ #define USER_PTRS_PER_PGD ((TASK_SIZE + (PGDIR_SIZE - 1)) / PGDIR_SIZE) #define PTRS_PER_PGD 1024 -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%08lx).\n", __FILE__, __LINE__, &(e), \ - pte_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %p(%08lx).\n", __FILE__, __LINE__, &(e), \ - pgd_val(e)) - static inline int pgd_needsync(pgd_t pgd) { return 0; } static inline void pgd_mkuptodate(pgd_t pgd) { } diff --git a/arch/um/include/asm/pgtable-4level.h b/arch/um/include/asm/pgtable-4level.h index 7a271b7b83d2..ff82f99c80fa 100644 --- a/arch/um/include/asm/pgtable-4level.h +++ b/arch/um/include/asm/pgtable-4level.h @@ -42,19 +42,6 @@ #define USER_PTRS_PER_PGD ((TASK_SIZE + (PGDIR_SIZE - 1)) / PGDIR_SIZE) -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pmd_val(e)) -#define pud_ERROR(e) \ - printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pud_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pgd_val(e)) - #define pud_none(x) (!(pud_val(x) & ~_PAGE_NEEDSYNC)) #define pud_bad(x) ((pud_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE) #define pud_present(x) (pud_val(x) & _PAGE_PRESENT) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 644ad02f7c9c..a02d953628a8 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -27,7 +27,6 @@ config X86_64 select ARCH_HAS_GIGANTIC_PAGE select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE select HAVE_ARCH_SOFT_DIRTY select MODULES_USE_ELF_RELA @@ -150,7 +149,6 @@ config X86 select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if X86_64 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if X86_64 - select ARCH_WANT_HUGETLB_VMEMMAP_PREINIT if X86_64 select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH select ARCH_WANT_IRQS_OFF_ACTIVATE_MM @@ -285,7 +283,6 @@ config X86 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select ASYNC_KERNEL_PGTABLE_FREE if IOMMU_SVA - select MMU_GATHER_RCU_TABLE_FREE select MMU_GATHER_MERGE_VMAS select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_REGS_AND_STACK_ACCESS_API @@ -1848,7 +1845,6 @@ config X86_USER_SHADOW_STACK bool "X86 userspace shadow stack" depends on AS_WRUSS depends on X86_64 - depends on PER_VMA_LOCK select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_USER_SHADOW_STACK select X86_CET diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h index bc3e549795c3..72a92cb9b53d 100644 --- a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h +++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h @@ -11,7 +11,7 @@ #undef CONFIG_PGTABLE_LEVELS #undef CONFIG_ILLEGAL_POINTER_VALUE #undef CONFIG_SPARSEMEM_VMEMMAP -#undef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#undef CONFIG_VMEMMAP_OPTIMIZATION #undef CONFIG_NR_CPUS #undef CONFIG_PARAVIRT_XXL diff --git a/arch/x86/events/intel/bts.c b/arch/x86/events/intel/bts.c index cbac54cb3a9e..5849392cf26d 100644 --- a/arch/x86/events/intel/bts.c +++ b/arch/x86/events/intel/bts.c @@ -66,9 +66,6 @@ static struct pmu bts_pmu; static int buf_nr_pages(struct page *page) { - if (!PagePrivate(page)) - return 1; - return 1 << page_private(page); } diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index 5754cd405562..49349afee611 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -781,8 +781,7 @@ static int topa_insert_pages(struct pt_buffer *buf, int cpu, gfp_t gfp) struct page *p; p = virt_to_page(buf->data_pages[buf->nr_pages]); - if (PagePrivate(p)) - order = page_private(p); + order = page_private(p); if (topa_table_full(topa)) { topa = topa_alloc(cpu, gfp); @@ -1296,8 +1295,7 @@ static int pt_buffer_try_single(struct pt_buffer *buf, int nr_pages) if (!intel_pt_validate_hw_cap(PT_CAP_single_range_output)) goto out; - if (PagePrivate(p)) - order = page_private(p); + order = page_private(p); if (1 << order != nr_pages) goto out; diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h index e9482a11ac52..83427765cfbf 100644 --- a/arch/x86/include/asm/pgtable-2level.h +++ b/arch/x86/include/asm/pgtable-2level.h @@ -2,11 +2,6 @@ #ifndef _ASM_X86_PGTABLE_2LEVEL_H #define _ASM_X86_PGTABLE_2LEVEL_H -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx\n", __FILE__, __LINE__, (e).pte_low) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx\n", __FILE__, __LINE__, pgd_val(e)) - /* * Certain architectures need to do special things when PTEs * within a page table are directly modified. Thus, the following diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h index dabafba957ea..d6729911e09a 100644 --- a/arch/x86/include/asm/pgtable-3level.h +++ b/arch/x86/include/asm/pgtable-3level.h @@ -8,17 +8,6 @@ * * Copyright (C) 1999 Ingo Molnar */ - -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %p(%08lx%08lx)\n", \ - __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %p(%016Lx)\n", \ - __FILE__, __LINE__, &(e), pmd_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %p(%016Lx)\n", \ - __FILE__, __LINE__, &(e), pgd_val(e)) - #define pxx_xchg64(_pxx, _ptr, _val) ({ \ _pxx##val_t *_p = (_pxx##val_t *)_ptr; \ _pxx##val_t _o = *_p; \ diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index ce45882ccd07..c861f3832bed 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -29,24 +29,6 @@ extern pgd_t init_top_pgt[]; extern void paging_init(void); static inline void sync_initial_page_table(void) { } -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pte_val(e)) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pmd_val(e)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pud_val(e)) - -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), p4d_val(e)) - -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pgd_val(e)) - struct mm_struct; #define mm_p4d_folded mm_p4d_folded diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 4362c26aa992..39271a5ea925 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -86,9 +86,8 @@ int set_pages_wb(struct page *page, int numpages); int set_pages_ro(struct page *page, int numpages); int set_pages_rw(struct page *page, int numpages); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); bool kernel_page_present(struct page *page); extern int kernel_set_to_readonly; diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 0ea680bf403a..2dae70c2b014 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -81,24 +81,77 @@ int strcmp(const char *cs, const char *ct); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1 void __memcpy_flushcache(void *dst, const void *src, size_t cnt); -static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) + +static __always_inline void movnti_4(void *dst, const void *src) { - if (__builtin_constant_p(cnt)) { - switch (cnt) { - case 4: - asm ("movntil %1, %0" : "=m"(*(u32 *)dst) : "r"(*(u32 *)src)); - return; - case 8: - asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src)); - return; - case 16: - asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src)); - asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src + 8))); - return; - } - } - __memcpy_flushcache(dst, src, cnt); + asm volatile("movntil %1, %0" + : "=m"(*(u32 *)dst) + : "r"(*(const u32 *)src) + : "memory"); } + +static __always_inline void movnti_8(void *dst, const void *src) +{ + asm volatile("movntiq %1, %0" + : "=m"(*(u64 *)dst) + : "r"(*(const u64 *)src) + : "memory"); +} + +static __always_inline void movnti_16(void *dst, const void *src) +{ + movnti_8(dst, src); + movnti_8(dst + 8, src + 8); +} + +static __always_inline void movnti_32(void *dst, const void *src) +{ + movnti_16(dst, src); + movnti_16(dst + 16, src + 16); +} + +static __always_inline void movnti_64(void *dst, const void *src) +{ + movnti_32(dst, src); + movnti_32(dst + 32, src + 32); +} + +static __always_inline void memcpy_flushcache(void *dst, const void *src, + size_t cnt) +{ + if (!__builtin_constant_p(cnt)) + return __memcpy_flushcache(dst, src, cnt); + + /* + * The relevant fixed-size copies here are the x86_64 struct page sizes: + * 64, 80, and 96 bytes. Keep 32-byte and 48-byte copies inline as well + * instead of sending those nearby fixed-size cases back to + * __memcpy_flushcache(). + */ + switch (cnt) { + case 4: movnti_4(dst, src); break; + case 8: movnti_8(dst, src); break; + case 16: movnti_16(dst, src); break; + case 32: movnti_32(dst, src); break; + case 48: movnti_32(dst, src); movnti_16(dst + 32, src + 32); break; + case 64: movnti_64(dst, src); break; + case 80: movnti_64(dst, src); movnti_16(dst + 64, src + 64); break; + case 96: movnti_64(dst, src); movnti_32(dst + 64, src + 64); break; + default: __memcpy_flushcache(dst, src, cnt); break; + } +} + +#define memcpy_nontemporal memcpy_nontemporal +/* + * Reuse the existing x86 flushcache backend as the non-temporal copy + * primitive. + */ +static __always_inline void memcpy_nontemporal(void *dst, const void *src, + size_t cnt) +{ + memcpy_flushcache(dst, src, cnt); +} + #endif #endif /* __KERNEL__ */ diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index c71f6774a2f3..b1d5c5a0ffb4 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -2700,24 +2700,16 @@ static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flag return __change_page_attr_set_clr(&cpa, 1); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_pages_np(page, 1, 0); -} - -int set_direct_map_default_noflush(struct page *page) -{ - return __set_pages_p(page, 1, 0); -} - -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - if (valid) - return __set_pages_p(page, nr, 0); - return __set_pages_np(page, nr, 0); } +int set_direct_map_default_noflush(struct page *page, unsigned int nr) +{ + return __set_pages_p(page, nr, 0); +} + #ifdef CONFIG_DEBUG_PAGEALLOC void __kernel_map_pages(struct page *page, int numpages, int enable) { diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h index f00a879dc298..60fb67a99729 100644 --- a/arch/xtensa/include/asm/pgtable.h +++ b/arch/xtensa/include/asm/pgtable.h @@ -204,10 +204,6 @@ */ #ifndef __ASSEMBLER__ -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd entry %08lx.\n", __FILE__, __LINE__, pgd_val(e)) #ifdef CONFIG_MMU extern pgd_t swapper_pg_dir[PAGE_SIZE/sizeof(pgd_t)]; diff --git a/arch/xtensa/include/asm/tlb.h b/arch/xtensa/include/asm/tlb.h index 8c3ceb427018..6fb7b78154f6 100644 --- a/arch/xtensa/include/asm/tlb.h +++ b/arch/xtensa/include/asm/tlb.h @@ -16,7 +16,7 @@ #include -#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, address) tlb_remove_ptdesc((tlb), page_ptdesc(pte)) void check_tlb_sanity(void); diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs index c1f4f635c2c0..e2d46b4d1a29 100644 --- a/drivers/android/binder/page_range.rs +++ b/drivers/android/binder/page_range.rs @@ -439,22 +439,9 @@ impl ShrinkablePageRange { // workqueue. let mm = MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?); { - let vma_read; - let mmap_read; - let vma = if let Some(ret) = mm.lock_vma_under_rcu(vma_addr) { - vma_read = ret; - check_vma(&vma_read, self) - } else { - mmap_read = mm.mmap_read_lock(); - mmap_read - .vma_lookup(vma_addr) - .and_then(|vma| check_vma(vma, self)) - }; - - match vma { - Some(vma) => vma.vm_insert_page(user_page_addr, &new_page)?, - None => return Err(ESRCH), - } + let vma_read_guard = mm.vma_start_read_unlocked(vma_addr).ok_or(ESRCH)?; + let vma = check_vma(&vma_read_guard, self).ok_or(ESRCH)?; + vma.vm_insert_page(user_page_addr, &new_page)?; } let inner = self.lock.lock(); diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index e4488ad86a65..d6eae0aa7085 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -259,21 +259,14 @@ static int binder_page_insert(struct binder_alloc *alloc, struct vm_area_struct *vma; int ret = -ESRCH; - /* attempt per-vma lock first */ - vma = lock_vma_under_rcu(mm, addr); - if (vma) { - if (binder_alloc_is_mapped(alloc)) - ret = vm_insert_page(vma, addr, page); - vma_end_read(vma); + vma = vma_start_read_unlocked(mm, addr); + if (!vma) return ret; - } - /* fall back to mmap_lock */ - mmap_read_lock(mm); - vma = vma_lookup(mm, addr); - if (vma && binder_alloc_is_mapped(alloc)) + if (binder_alloc_is_mapped(alloc)) ret = vm_insert_page(vma, addr, page); - mmap_read_unlock(mm); + + vma_end_read(vma); return ret; } @@ -1142,7 +1135,6 @@ enum lru_status binder_alloc_free_page(struct list_head *item, struct vm_area_struct *vma; struct page *page_to_free; unsigned long page_addr; - int mm_locked = 0; size_t index; if (!mmget_not_zero(mm)) @@ -1151,27 +1143,25 @@ enum lru_status binder_alloc_free_page(struct list_head *item, index = mdata->page_index; page_addr = alloc->vm_start + index * PAGE_SIZE; - /* attempt per-vma lock first */ + /* + * Attempt per-vma lock. This is essentially a + * "trylock". It can fail even if the VMA exists + * for 'page_addr'. + */ vma = lock_vma_under_rcu(mm, page_addr); if (!vma) { - /* fall back to mmap_lock */ - if (!mmap_read_trylock(mm)) - goto err_mmap_read_lock_failed; - mm_locked = 1; - vma = vma_lookup(mm, page_addr); + /* + * If the vma exists, we can't continue because we cannot + * remove the page from the vma. However, if the vma was + * unmapped, it's okay to continue. + */ + if (binder_alloc_is_mapped(alloc)) + goto err_vma_lock_failed; } if (!mutex_trylock(&alloc->mutex)) goto err_get_alloc_mutex_failed; - /* - * Since a binder_alloc can only be mapped once, we ensure - * the vma corresponds to this mapping by checking whether - * the binder_alloc is still mapped. - */ - if (vma && !binder_alloc_is_mapped(alloc)) - goto err_invalid_vma; - trace_binder_unmap_kernel_start(alloc, index); page_to_free = alloc->pages[index]; @@ -1182,7 +1172,12 @@ enum lru_status binder_alloc_free_page(struct list_head *item, list_lru_isolate(lru, item); spin_unlock(&lru->lock); - if (vma) { + /* + * Since a binder_alloc can only be mapped once, we ensure + * the vma corresponds to this mapping by checking whether + * the binder_alloc is still mapped. + */ + if (vma && binder_alloc_is_mapped(alloc)) { trace_binder_unmap_user_start(alloc, index); zap_vma_range(vma, page_addr, PAGE_SIZE); @@ -1191,23 +1186,17 @@ enum lru_status binder_alloc_free_page(struct list_head *item, } mutex_unlock(&alloc->mutex); - if (mm_locked) - mmap_read_unlock(mm); - else + if (vma) vma_end_read(vma); mmput_async(mm); binder_free_page(page_to_free); return LRU_REMOVED_RETRY; -err_invalid_vma: - mutex_unlock(&alloc->mutex); err_get_alloc_mutex_failed: - if (mm_locked) - mmap_read_unlock(mm); - else + if (vma) vma_end_read(vma); -err_mmap_read_lock_failed: +err_vma_lock_failed: mmput_async(mm); err_mmget: return LRU_SKIP; diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index 974c4691887e..028e2f0f587e 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -158,17 +160,32 @@ int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, } int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm, - const void *src, unsigned int src_len, void *dst) + struct scatterlist *sg, unsigned int src_len, void *dst) { struct zcomp_req req = { - .src = src, .dst = dst, .src_len = src_len, .dst_len = PAGE_SIZE, }; + void *src = NULL; + int ret; might_sleep(); - return comp->ops->decompress(comp->params, &zstrm->ctx, &req); + + if (sg_is_last(sg)) { + /* the object is contained within one page, read it in-place */ + src = kmap_local_page(sg_page(sg)); + req.src = src + sg->offset; + } else { + /* the object spans two pages, linearize it into local copy */ + sg_copy_to_buffer(sg, 2, zstrm->local_copy, src_len); + req.src = zstrm->local_copy; + } + + ret = comp->ops->decompress(comp->params, &zstrm->ctx, &req); + if (src) + kunmap_local(src); + return ret; } int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index 81a0f3f6ff48..f38fd31f9e4e 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -5,6 +5,8 @@ #include +struct scatterlist; + #define ZCOMP_PARAM_NOT_SET INT_MIN struct deflate_params { @@ -91,6 +93,6 @@ void zcomp_stream_put(struct zcomp_strm *zstrm); int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, const void *src, unsigned int *dst_len); int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm, - const void *src, unsigned int src_len, void *dst); + struct scatterlist *sg, unsigned int src_len, void *dst); #endif /* _ZCOMP_H_ */ diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index a9b3bb1d3bef..024402438bd1 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -415,7 +416,7 @@ static ssize_t mem_used_max_store(struct device *dev, * Mark all pages which are older than or equal to cutoff as IDLE. * Callers should hold the zram init lock in read mode */ -static void mark_idle(struct zram *zram, ktime_t cutoff) +static void mark_idle(struct zram *zram, time64_t cutoff) { int is_idle = 1; unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; @@ -439,7 +440,7 @@ static void mark_idle(struct zram *zram, ktime_t cutoff) #ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME is_idle = !cutoff || - ktime_after(cutoff, zram->table[index].attr.ac_time); + cutoff > zram->table[index].attr.ac_time; #endif if (is_idle) set_slot_flag(zram, index, ZRAM_IDLE); @@ -453,21 +454,26 @@ static ssize_t idle_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); - ktime_t cutoff = 0; + time64_t cutoff = 0; if (!sysfs_streq(buf, "all")) { /* * If it did not parse as 'all' try to treat it as an integer * when we have memory tracking enabled. */ + time64_t uptime; u32 age_sec; - if (IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) && - !kstrtouint(buf, 0, &age_sec)) - cutoff = ktime_sub((u32)ktime_get_boottime_seconds(), - age_sec); - else + if (!IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) || + kstrtouint(buf, 0, &age_sec)) return -EINVAL; + + /* No slot can be older than the system uptime */ + uptime = ktime_get_boottime_seconds(); + if (age_sec >= uptime) + return len; + + cutoff = uptime - age_sec; } guard(rwsem_read)(&zram->dev_lock); @@ -1342,9 +1348,9 @@ static int decompress_bdev_page(struct zram *zram, struct page *page, unsigned long index) { struct zcomp_strm *zstrm; + struct scatterlist sg[1]; unsigned int size; int ret, prio; - void *src; slot_lock(zram, index); /* Since slot was unlocked we need to make sure it's still ZRAM_WB */ @@ -1363,13 +1369,18 @@ static int decompress_bdev_page(struct zram *zram, struct page *page, size = get_slot_size(zram, index); prio = get_slot_comp_priority(zram, index); + sg_init_table(sg, 1); + sg_set_page(sg, page, size, 0); + zstrm = zcomp_stream_get(zram->comps[prio]); - src = kmap_local_page(page); - ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, + ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size, zstrm->local_copy); - if (!ret) - copy_page(src, zstrm->local_copy); - kunmap_local(src); + if (!ret) { + void *dst = kmap_local_page(page); + + copy_page(dst, zstrm->local_copy); + kunmap_local(dst); + } zcomp_stream_put(zstrm); slot_unlock(zram, index); @@ -1717,11 +1728,6 @@ static int comp_params_store(struct zram *zram, u32 prio, s32 level, dict_path, sz); return sz; } - if (sz == 0) { - pr_err("failed to load dictionary %s (empty file)\n", - dict_path); - return -EINVAL; - } } zram->params[prio].dict_sz = sz; @@ -2086,15 +2092,19 @@ static int read_same_filled_page(struct zram *zram, struct page *page, static int read_incompressible_page(struct zram *zram, struct page *page, unsigned long index) { + struct scatterlist sg[2]; unsigned long handle; void *src, *dst; handle = get_slot_handle(zram, index); - src = zs_obj_read_begin(zram->mem_pool, handle, PAGE_SIZE, NULL); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, PAGE_SIZE); + /* an incompressible object never spans two pages */ + src = kmap_local_page(sg_page(sg)); dst = kmap_local_page(page); copy_page(dst, src); kunmap_local(dst); - zs_obj_read_end(zram->mem_pool, handle, PAGE_SIZE, src); + kunmap_local(src); + zs_obj_read_sg_end(zram->mem_pool, handle); return 0; } @@ -2103,9 +2113,10 @@ static int read_compressed_page(struct zram *zram, struct page *page, unsigned long index) { struct zcomp_strm *zstrm; + struct scatterlist sg[2]; unsigned long handle; unsigned int size; - void *src, *dst; + void *dst; int ret, prio; handle = get_slot_handle(zram, index); @@ -2113,12 +2124,11 @@ static int read_compressed_page(struct zram *zram, struct page *page, prio = get_slot_comp_priority(zram, index); zstrm = zcomp_stream_get(zram->comps[prio]); - src = zs_obj_read_begin(zram->mem_pool, handle, size, - zstrm->local_copy); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size); dst = kmap_local_page(page); - ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, dst); + ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size, dst); kunmap_local(dst); - zs_obj_read_end(zram->mem_pool, handle, size, src); + zs_obj_read_sg_end(zram->mem_pool, handle); zcomp_stream_put(zstrm); return ret; @@ -2128,25 +2138,23 @@ static int read_compressed_page(struct zram *zram, struct page *page, static int read_from_zspool_raw(struct zram *zram, struct page *page, unsigned long index) { - struct zcomp_strm *zstrm; + struct scatterlist sg[2]; unsigned long handle; unsigned int size; - void *src; + void *dst; handle = get_slot_handle(zram, index); size = get_slot_size(zram, index); /* - * We need to get stream just for ->local_copy buffer, in - * case if object spans two physical pages. No decompression - * takes place here, as we read raw compressed data. + * No decompression takes place here, we copy out raw compressed + * data directly into the destination page. */ - zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]); - src = zs_obj_read_begin(zram->mem_pool, handle, size, - zstrm->local_copy); - memcpy_to_page(page, 0, src, size); - zs_obj_read_end(zram->mem_pool, handle, size, src); - zcomp_stream_put(zstrm); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size); + dst = kmap_local_page(page); + sg_copy_to_buffer(sg, sg_nents(sg), dst, size); + kunmap_local(dst); + zs_obj_read_sg_end(zram->mem_pool, handle); memzero_page(page, size, PAGE_SIZE - size); diff --git a/drivers/char/Makefile b/drivers/char/Makefile index a46d7bf7c4c8..bb3bf66937b3 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -3,7 +3,7 @@ # Makefile for the kernel character device drivers. # -obj-y += mem.o random.o +obj-y += random.o obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o obj-y += misc.o obj-$(CONFIG_TEST_MISC_MINOR) += misc_minor_kunit.o diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index a71ddaa810a0..b4fd482d39d2 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -944,7 +944,7 @@ static int hid_scan_report(struct hid_device *hid) hid_parser_reserved }; - struct hid_parser *parser __free(kvfree) = vzalloc(sizeof(*parser)); + struct hid_parser *parser __free(vfree) = vzalloc(sizeof(*parser)); if (!parser) return -ENOMEM; @@ -1265,7 +1265,7 @@ static int hid_parse_collections(struct hid_device *device) hid_parser_reserved }; - struct hid_parser *parser __free(kvfree) = vzalloc(sizeof(*parser)); + struct hid_parser *parser __free(vfree) = vzalloc(sizeof(*parser)); if (!parser) return -ENOMEM; diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index b8325cb09a37..8adf6ddca3d6 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -516,7 +516,8 @@ static void end_bitmap_write(struct bio *bio) static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) { - struct buffer_head *bh = page_buffers(page); + struct folio *folio = page_folio(page); + struct buffer_head *bh = folio_buffers(folio); while (bh && bh->b_blocknr) { atomic_inc(&bitmap->pending_writes); @@ -533,19 +534,15 @@ static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) static void free_buffers(struct page *page) { - struct buffer_head *bh; + struct folio *folio = page_folio(page); + struct buffer_head *bh = folio_detach_private(folio); - if (!PagePrivate(page)) - return; - - bh = page_buffers(page); while (bh) { struct buffer_head *next = bh->b_this_page; free_buffer_head(bh); bh = next; } - detach_page_private(page); - put_page(page); + folio_put(folio); } /* read a page from a file. @@ -560,6 +557,7 @@ static int read_file_page(struct file *file, unsigned long index, { int ret = 0; struct inode *inode = file_inode(file); + struct folio *folio = page_folio(page); struct buffer_head *bh; sector_t block, blk_cur; unsigned long blocksize = i_blocksize(inode); @@ -567,12 +565,12 @@ static int read_file_page(struct file *file, unsigned long index, pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE, (unsigned long long)index << PAGE_SHIFT); - bh = alloc_page_buffers(page, blocksize); + bh = folio_alloc_buffers(folio, blocksize, GFP_NOFS | __GFP_ACCOUNT); if (!bh) { ret = -ENOMEM; goto out; } - attach_page_private(page, bh); + folio_attach_private(folio, bh); blk_cur = index << (PAGE_SHIFT - inode->i_blkbits); while (bh) { block = blk_cur; diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 30a51c365ce8..5fb86595e8bd 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -80,7 +80,7 @@ static void pmem_mkpage_present(struct pmem_device *pmem, phys_addr_t offset, * here since we're in the driver I/O path and * outstanding I/O requests pin the dev_pagemap. */ - if (test_and_clear_pmem_poison(page)) + if (TestClearPageHWPoison(page)) clear_mce_nospec(pfn); } } diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h index a48509f90196..76870505dd79 100644 --- a/drivers/nvdimm/pmem.h +++ b/drivers/nvdimm/pmem.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NVDIMM_PMEM_H__ #define __NVDIMM_PMEM_H__ -#include #include #include #include @@ -31,15 +30,4 @@ long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, void **kaddr, unsigned long *pfn); -#ifdef CONFIG_MEMORY_FAILURE -static inline bool test_and_clear_pmem_poison(struct page *page) -{ - return TestClearPageHWPoison(page); -} -#else -static inline bool test_and_clear_pmem_poison(struct page *page) -{ - return false; -} -#endif #endif /* __NVDIMM_PMEM_H__ */ diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index 076c1b0ab87f..f75b5cc71cfc 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c @@ -863,10 +863,10 @@ EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames); int gnttab_pages_set_private(int nr_pages, struct page **pages) { +#if BITS_PER_LONG < 64 int i; for (i = 0; i < nr_pages; i++) { -#if BITS_PER_LONG < 64 struct xen_page_foreign *foreign; foreign = kzalloc_obj(*foreign); @@ -874,9 +874,9 @@ int gnttab_pages_set_private(int nr_pages, struct page **pages) return -ENOMEM; set_page_private(pages[i], (unsigned long)foreign); -#endif - SetPagePrivate(pages[i]); } +#endif + /* Data is stored in page->private on 64-bit */ return 0; } @@ -1031,12 +1031,11 @@ void gnttab_pages_clear_private(int nr_pages, struct page **pages) int i; for (i = 0; i < nr_pages; i++) { - if (PagePrivate(pages[i])) { #if BITS_PER_LONG < 64 + if (page_private(pages[i])) kfree((void *)page_private(pages[i])); #endif - ClearPagePrivate(pages[i]); - } + set_page_private(pages[i], 0); } } EXPORT_SYMBOL_GPL(gnttab_pages_clear_private); diff --git a/fs/Kconfig b/fs/Kconfig index 46313f65ee54..70a1d0760d28 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -278,7 +278,7 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP def_bool HUGETLB_PAGE depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP depends on SPARSEMEM_VMEMMAP - select SPARSEMEM_VMEMMAP_PREINIT if ARCH_WANT_HUGETLB_VMEMMAP_PREINIT + select VMEMMAP_OPTIMIZATION config HUGETLB_PMD_PAGE_TABLE_SHARING def_bool HUGETLB_PAGE diff --git a/fs/buffer.c b/fs/buffer.c index 1dc933ba6925..5d3852b726d6 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -779,14 +779,6 @@ no_grow: } EXPORT_SYMBOL_GPL(folio_alloc_buffers); -struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size) -{ - gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT; - - return folio_alloc_buffers(page_folio(page), size, gfp); -} -EXPORT_SYMBOL_GPL(alloc_page_buffers); - static inline void link_dev_buffers(struct folio *folio, struct buffer_head *head) { diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 42b55ce30a32..21d52d849842 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -70,9 +70,7 @@ static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned static inline struct ceph_snap_context *page_snap_context(struct page *page) { - if (PagePrivate(page)) - return (void *)page->private; - return NULL; + return (void *)page_private(page); } /* @@ -124,8 +122,8 @@ static bool ceph_dirty_folio(struct address_space *mapping, struct folio *folio) spin_unlock(&ci->i_ceph_lock); /* - * Reference snap context in folio->private. Also set - * PagePrivate so that we get invalidate_folio callback. + * Reference snap context in folio->private. Setting folio->private is + * what gets us the invalidate_folio callback. */ VM_WARN_ON_FOLIO(folio->private, folio); folio_attach_private(folio, snapc); diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 8f212e7f39f6..b6a253217a5a 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -65,7 +65,6 @@ void fscrypt_free_bounce_page(struct page *bounce_page) if (!bounce_page) return; set_page_private(bounce_page, (unsigned long)NULL); - ClearPagePrivate(bounce_page); mempool_free(bounce_page, fscrypt_bounce_page_pool); } EXPORT_SYMBOL(fscrypt_free_bounce_page); @@ -215,7 +214,6 @@ struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio, return ERR_PTR(err); } } - SetPagePrivate(ciphertext_page); set_page_private(ciphertext_page, (unsigned long)folio); return ciphertext_page; } diff --git a/fs/erofs/data.c b/fs/erofs/data.c index f09c152d5516..968303afd620 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -242,19 +242,23 @@ int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map) /* * bit 30: I/O error occurred on this folio * bit 29: CPU has dirty data in D-cache (needs aliasing handling); - * bit 0 - 29: remaining parts to complete this folio + * bit 0 - 28: remaining parts to complete this folio, biased by 1 so that + * ->private stays non-NULL while the folio is attached */ #define EROFS_ONLINEFOLIO_EIO 30 #define EROFS_ONLINEFOLIO_DIRTY 29 +#define EROFS_ONLINEFOLIO_COUNT_MASK (BIT(EROFS_ONLINEFOLIO_DIRTY) - 1) +#define EROFS_ONLINEFOLIO_BIAS 1 void erofs_onlinefolio_init(struct folio *folio) { union { atomic_t o; void *v; - } u = { .o = ATOMIC_INIT(1) }; + } u = { .o = ATOMIC_INIT(1 + EROFS_ONLINEFOLIO_BIAS) }; - folio->private = u.v; /* valid only if file-backed folio is locked */ + /* valid only if file-backed folio is locked */ + folio_attach_private(folio, u.v); } void erofs_onlinefolio_split(struct folio *folio) @@ -268,14 +272,14 @@ void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty) do { orig = atomic_read((atomic_t *)&folio->private); - DBG_BUGON(orig <= 0); + DBG_BUGON((orig & EROFS_ONLINEFOLIO_COUNT_MASK) <= EROFS_ONLINEFOLIO_BIAS); v = dirty << EROFS_ONLINEFOLIO_DIRTY; v |= (orig - 1) | (!!err << EROFS_ONLINEFOLIO_EIO); } while (atomic_cmpxchg((atomic_t *)&folio->private, orig, v) != orig); - if (v & (BIT(EROFS_ONLINEFOLIO_DIRTY) - 1)) + if ((v & EROFS_ONLINEFOLIO_COUNT_MASK) != EROFS_ONLINEFOLIO_BIAS) return; - folio->private = 0; + folio_detach_private(folio); if (v & BIT(EROFS_ONLINEFOLIO_DIRTY)) flush_dcache_folio(folio); folio_end_read(folio, !(v & BIT(EROFS_ONLINEFOLIO_EIO))); diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 6b07e73ee2aa..e981e371d6c2 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -1886,21 +1886,14 @@ static void z_erofs_readahead(struct readahead_control *rac) struct inode *realinode = erofs_real_inode(sharedinode, &need_iput); Z_EROFS_DEFINE_FRONTEND(f, realinode, sharedinode, readahead_pos(rac)); unsigned int nrpages = readahead_count(rac); - struct folio *head = NULL, *folio; + struct folio *folio; int err; trace_erofs_readahead(realinode, readahead_index(rac), nrpages, false); z_erofs_pcluster_readmore(&f, rac, true); - while ((folio = readahead_folio(rac))) { - folio->private = head; - head = folio; - } - - /* traverse in reverse order for best metadata I/O performance */ - while (head) { - folio = head; - head = folio_get_private(folio); + /* traverse from last to first for best metadata I/O performance */ + while ((folio = readahead_folio_last(rac))) { err = z_erofs_scan_folio(&f, folio, true); if (err && err != -EINTR) erofs_err(realinode->i_sb, "readahead error at folio %lu @ nid %llu", diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index ce88092d9ce2..09d9b8d0fdcc 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1064,13 +1064,15 @@ static void cancel_cluster_writeback(struct compress_ctx *cc, /* Cancel writeback and stay locked. */ for (i = 0; i < cc->cluster_size; i++) { + struct folio *folio = page_folio(cc->rpages[i]); + if (i < submitted) { inode_inc_dirty_pages(cc->inode); - lock_page(cc->rpages[i]); + folio_lock(folio); } - clear_page_private_gcing(cc->rpages[i]); - if (folio_test_writeback(page_folio(cc->rpages[i]))) - end_page_writeback(cc->rpages[i]); + folio_clear_f2fs_gcing(folio); + if (folio_test_writeback(folio)) + folio_end_writeback(folio); } } @@ -1078,11 +1080,15 @@ static void set_cluster_dirty(struct compress_ctx *cc) { int i; - for (i = 0; i < cc->cluster_size; i++) - if (cc->rpages[i]) { - set_page_dirty(cc->rpages[i]); - set_page_private_gcing(cc->rpages[i]); - } + for (i = 0; i < cc->cluster_size; i++) { + struct folio *folio; + + if (!cc->rpages[i]) + continue; + folio = page_folio(cc->rpages[i]); + folio_mark_dirty(folio); + folio_set_f2fs_gcing(folio); + } } static int prepare_compress_overwrite(struct compress_ctx *cc, @@ -1281,7 +1287,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, .op = REQ_OP_WRITE, .op_flags = wbc_to_write_flags(wbc), .old_blkaddr = NEW_ADDR, - .page = NULL, + .folio = NULL, .encrypted_page = NULL, .compressed_page = NULL, .io_type = io_type, @@ -1370,7 +1376,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, block_t blkaddr; blkaddr = f2fs_data_blkaddr(&dn); - fio.page = cc->rpages[i]; + fio.folio = page_folio(cc->rpages[i]); fio.old_blkaddr = blkaddr; /* cluster header */ @@ -1476,9 +1482,12 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) } for (i = 0; i < cic->nr_rpages; i++) { + struct folio *rfolio; + WARN_ON(!cic->rpages[i]); - clear_page_private_gcing(cic->rpages[i]); - end_page_writeback(cic->rpages[i]); + rfolio = page_folio(cic->rpages[i]); + folio_clear_f2fs_gcing(rfolio); + folio_end_writeback(rfolio); } page_array_free(sbi, cic->rpages, cic->nr_rpages); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 21f396ebe22c..ca8232a9095f 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2923,7 +2923,7 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) return true; if (fio) { - if (page_private_gcing(fio->page)) + if (folio_test_f2fs_gcing(fio->folio)) return true; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 128813d4f6e3..e3108c5f7c91 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1357,10 +1357,7 @@ struct f2fs_io_info { blk_opf_t op_flags; /* req_flag_bits */ block_t new_blkaddr; /* new block address to be written */ block_t old_blkaddr; /* old block address before Cow */ - union { - struct page *page; /* page to be written */ - struct folio *folio; - }; + struct folio *folio; /* folio to be written */ struct page *encrypted_page; /* encrypted page */ struct page *compressed_page; /* compressed page */ struct list_head list; /* serialize IOs */ @@ -1613,27 +1610,27 @@ static inline void f2fs_set_bit(unsigned int nr, char *addr); static inline void f2fs_clear_bit(unsigned int nr, char *addr); /* - * Layout of f2fs page.private: + * Layout of f2fs folio->private: * * Layout A: lowest bit should be 1 * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... | - * bit 0 PAGE_PRIVATE_NOT_POINTER - * bit 1 PAGE_PRIVATE_ONGOING_MIGRATION - * bit 2 PAGE_PRIVATE_INLINE_INODE - * bit 3 PAGE_PRIVATE_REF_RESOURCE - * bit 4 PAGE_PRIVATE_ATOMIC_WRITE + * bit 0 F2FS_FOLIO_PRIVATE_NOT_POINTER + * bit 1 F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION + * bit 2 F2FS_FOLIO_PRIVATE_INLINE_INODE + * bit 3 F2FS_FOLIO_PRIVATE_REF_RESOURCE + * bit 4 F2FS_FOLIO_PRIVATE_ATOMIC_WRITE * bit 5- f2fs private data * * Layout B: lowest bit should be 0 - * page.private is a wrapped pointer. + * folio->private is a wrapped pointer. */ enum { - PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ - PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ - PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */ - PAGE_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */ - PAGE_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */ - PAGE_PRIVATE_MAX + F2FS_FOLIO_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ + F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ + F2FS_FOLIO_PRIVATE_INLINE_INODE, /* inode page contains inline data */ + F2FS_FOLIO_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */ + F2FS_FOLIO_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */ + F2FS_FOLIO_PRIVATE_MAX }; /* For compression */ @@ -2681,86 +2678,68 @@ release_quota: return -ENOSPC; } -#define PAGE_PRIVATE_GET_FUNC(name, flagname) \ +#define F2FS_FOLIO_PRIVATE_GET_FUNC(name, flagname) \ static inline bool folio_test_f2fs_##name(const struct folio *folio) \ { \ unsigned long priv = (unsigned long)folio->private; \ - unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \ - (1UL << PAGE_PRIVATE_##flagname); \ + unsigned long v = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) | \ + (1UL << F2FS_FOLIO_PRIVATE_##flagname); \ return (priv & v) == v; \ -} \ -static inline bool page_private_##name(struct page *page) \ -{ \ - return PagePrivate(page) && \ - test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \ - test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ } -#define PAGE_PRIVATE_SET_FUNC(name, flagname) \ +#define F2FS_FOLIO_PRIVATE_SET_FUNC(name, flagname) \ static inline void folio_set_f2fs_##name(struct folio *folio) \ { \ - unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \ - (1UL << PAGE_PRIVATE_##flagname); \ + unsigned long v = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) | \ + (1UL << F2FS_FOLIO_PRIVATE_##flagname); \ if (!folio->private) \ folio_attach_private(folio, (void *)v); \ else { \ v |= (unsigned long)folio->private; \ folio->private = (void *)v; \ } \ -} \ -static inline void set_page_private_##name(struct page *page) \ -{ \ - if (!PagePrivate(page)) \ - attach_page_private(page, (void *)0); \ - set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \ - set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ } -#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \ +#define F2FS_FOLIO_PRIVATE_CLEAR_FUNC(name, flagname) \ static inline void folio_clear_f2fs_##name(struct folio *folio) \ { \ unsigned long v = (unsigned long)folio->private; \ \ - v &= ~(1UL << PAGE_PRIVATE_##flagname); \ - if (v == (1UL << PAGE_PRIVATE_NOT_POINTER)) \ + v &= ~(1UL << F2FS_FOLIO_PRIVATE_##flagname); \ + if (v == (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER)) \ folio_detach_private(folio); \ else \ folio->private = (void *)v; \ -} \ -static inline void clear_page_private_##name(struct page *page) \ -{ \ - clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ - if (page_private(page) == BIT(PAGE_PRIVATE_NOT_POINTER)) \ - detach_page_private(page); \ } -PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); -PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE); -PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE); +F2FS_FOLIO_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); +F2FS_FOLIO_PRIVATE_GET_FUNC(inline, INLINE_INODE); +F2FS_FOLIO_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); +F2FS_FOLIO_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE); -PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE); -PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE); -PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE); +F2FS_FOLIO_PRIVATE_SET_FUNC(reference, REF_RESOURCE); +F2FS_FOLIO_PRIVATE_SET_FUNC(inline, INLINE_INODE); +F2FS_FOLIO_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); +F2FS_FOLIO_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE); -PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); -PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); -PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE); static inline unsigned long folio_get_f2fs_data(struct folio *folio) { unsigned long data = (unsigned long)folio->private; - if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data)) + if (!test_bit(F2FS_FOLIO_PRIVATE_NOT_POINTER, &data)) return 0; - return data >> PAGE_PRIVATE_MAX; + return data >> F2FS_FOLIO_PRIVATE_MAX; } static inline void folio_set_f2fs_data(struct folio *folio, unsigned long data) { - data = (1UL << PAGE_PRIVATE_NOT_POINTER) | (data << PAGE_PRIVATE_MAX); + data = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) | + (data << F2FS_FOLIO_PRIVATE_MAX); if (!folio_test_private(folio)) folio_attach_private(folio, (void *)data); diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 63b712d3d599..8c156e1fd37d 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3803,7 +3803,7 @@ static int __get_segment_type_6(struct f2fs_io_info *fio) if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) return CURSEG_COLD_DATA_PINNED; - if (page_private_gcing(fio->page)) { + if (folio_test_f2fs_gcing(fio->folio)) { if (fio->sbi->am.atgc_enabled && (fio->io_type == FS_DATA_IO) && (fio->sbi->gc_mode != GC_URGENT_HIGH) && diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 678f4c5329e7..1e0de37b2411 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -2077,8 +2077,7 @@ iomap_writepages(struct iomap_writepage_ctx *wpc) * Writeback from reclaim context should never happen except in the case * of a VM regression so warn about it and refuse to write the data. */ - if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC | PF_KSWAPD)) == - PF_MEMALLOC)) + if (WARN_ON_ONCE((current->flags & PF_MEMALLOC))) return -EIO; while ((folio = writeback_iter(mapping, wpc->wbc, folio, &error))) { diff --git a/fs/nfs/file.c b/fs/nfs/file.c index e1bdd10b35f1..38f830a6467c 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -484,7 +484,7 @@ static int nfs_write_end(const struct kiocb *iocb, * Partially or wholly invalidate a page * - Release the private state associated with a page if undergoing complete * page invalidation - * - Called if either PG_private or PG_fscache is set on the page + * - Called if either folio->private or PG_fscache is set on the page * - Caller holds page lock */ static void nfs_invalidate_folio(struct folio *folio, size_t offset, @@ -555,7 +555,7 @@ static void nfs_check_dirty_writeback(struct folio *folio, * Attempt to clear the private state associated with a page when an error * occurs that requires the cached contents of an inode to be written back or * destroyed - * - Called if either PG_private or fscache is set on the page + * - Called if either page->private or fscache is set on the page * - Caller holds page lock * - Return 0 if successful, -error otherwise */ diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 623e7ef1f73d..b6967b528669 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -717,7 +717,6 @@ static void nfs_inode_add_request(struct nfs_page *req) nfs_lock_request(req); spin_lock(&mapping->i_private_lock); set_bit(PG_MAPPED, &req->wb_flags); - folio_set_private(folio); folio->private = req; spin_unlock(&mapping->i_private_lock); atomic_long_inc(&nfsi->nrequests); @@ -745,7 +744,6 @@ static void nfs_inode_remove_request(struct nfs_page *req) spin_lock(&mapping->i_private_lock); folio->private = NULL; - folio_clear_private(folio); clear_bit(PG_MAPPED, &req->wb_head->wb_flags); spin_unlock(&mapping->i_private_lock); diff --git a/fs/proc/base.c b/fs/proc/base.c index 5a18636ba81f..3dff299bfe1b 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -594,7 +594,8 @@ static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns, * exporting for a long time so userspace might depend on it. */ if (badness != LONG_MIN) - points = (1000 + badness * 1000 / (long)totalpages) * 2 / 3; + points = (OOM_SCORE_ADJ_MAX + + badness * OOM_SCORE_ADJ_MAX / (long)totalpages) * 2 / 3; seq_printf(m, "%lu\n", points); diff --git a/fs/proc/internal.h b/fs/proc/internal.h index b9aaac41c283..df873df9afc5 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -385,10 +385,8 @@ struct mem_size_stats; struct proc_maps_locking_ctx { struct mm_struct *mm; -#ifdef CONFIG_PER_VMA_LOCK bool mmap_locked; struct vm_area_struct *locked_vma; -#endif }; struct proc_maps_private { diff --git a/fs/proc/page.c b/fs/proc/page.c index 260772b20bd9..f90e1030825e 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -232,7 +232,6 @@ u64 stable_page_flags(const struct page *page) u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved); u |= kpf_copy_bit(k, KPF_OWNER_2, PG_owner_2); - u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private); u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2); u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1); u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 5c54aebe2118..ee9fc2fd2b2f 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -130,30 +130,12 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif -#ifdef CONFIG_PER_VMA_LOCK - -static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) -{ - int ret = mmap_read_lock_killable(lock_ctx->mm); - - if (!ret) - lock_ctx->mmap_locked = true; - - return ret; -} - -static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +static void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) { mmap_read_unlock(lock_ctx->mm); lock_ctx->mmap_locked = false; } -static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) -{ - lock_ctx->locked_vma = NULL; - lock_ctx->mmap_locked = false; -} - static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) { if (lock_ctx->locked_vma) { @@ -162,23 +144,10 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static inline bool lock_vma_range(struct seq_file *m, - struct proc_maps_locking_ctx *lock_ctx) +static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) { - rcu_read_lock(); - reset_lock_ctx(lock_ctx); - - return true; -} - -static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) -{ - if (lock_ctx->mmap_locked) { - unlock_ctx_mm(lock_ctx); - } else { - unlock_ctx_vma(lock_ctx); - rcu_read_unlock(); - } + lock_ctx->locked_vma = NULL; + lock_ctx->mmap_locked = false; } static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, @@ -198,8 +167,8 @@ static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, return vma; } -static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, - loff_t pos) +static bool fallback_to_mmap_lock(struct proc_maps_private *priv, + loff_t pos) { struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; @@ -215,7 +184,8 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return true; } -static inline void drop_rcu(struct proc_maps_private *priv) +#if defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA) +static void drop_rcu(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) return; @@ -223,7 +193,7 @@ static inline void drop_rcu(struct proc_maps_private *priv) rcu_read_unlock(); } -static inline void reacquire_rcu(struct proc_maps_private *priv) +static void reacquire_rcu(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) return; @@ -232,46 +202,7 @@ static inline void reacquire_rcu(struct proc_maps_private *priv) /* Reinitialize the iterator. */ vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); } - -#else /* CONFIG_PER_VMA_LOCK */ - -static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) -{ - return mmap_read_lock_killable(lock_ctx->mm); -} - -static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) -{ - mmap_read_unlock(lock_ctx->mm); -} - -static inline bool lock_vma_range(struct seq_file *m, - struct proc_maps_locking_ctx *lock_ctx) -{ - return lock_ctx_mm(lock_ctx) == 0; -} - -static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) -{ - unlock_ctx_mm(lock_ctx); -} - -static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, - loff_t last_pos) -{ - return vma_next(&priv->iter); -} - -static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, - loff_t pos) -{ - return false; -} - -static inline void drop_rcu(struct proc_maps_private *priv) {} -static inline void reacquire_rcu(struct proc_maps_private *priv) {} - -#endif /* CONFIG_PER_VMA_LOCK */ +#endif static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) { @@ -297,9 +228,6 @@ retry: * found the extended vma with the same vm_start. */ *ppos = vma->vm_end; - } else { - *ppos = SENTINEL_VMA_GATE; - vma = get_gate_vma(priv->lock_ctx.mm); } return vma; @@ -309,6 +237,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos) { struct proc_maps_private *priv = m->private; struct proc_maps_locking_ctx *lock_ctx; + struct vm_area_struct *vma; loff_t last_addr = *ppos; struct mm_struct *mm; @@ -328,13 +257,8 @@ static void *m_start(struct seq_file *m, loff_t *ppos) return NULL; } - if (!lock_vma_range(m, lock_ctx)) { - mmput(mm); - put_task_struct(priv->task); - priv->task = NULL; - return ERR_PTR(-EINTR); - } - + rcu_read_lock(); + reset_lock_ctx(lock_ctx); /* * Reset current position if last_addr was set before * and it's not a sentinel. @@ -343,19 +267,39 @@ static void *m_start(struct seq_file *m, loff_t *ppos) *ppos = last_addr = priv->last_pos; vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); hold_task_mempolicy(priv); + /* + * If seq_file had to flush its collected data right after m_next() set + * position to SENTINEL_VMA_GATE, m_start() will get that sentinel and + * should return gate_vma without calling proc_get_vma(). + */ if (last_addr == SENTINEL_VMA_GATE) return get_gate_vma(mm); - return proc_get_vma(m, ppos); + vma = proc_get_vma(m, ppos); + if (vma) + return vma; + + /* Return gate VMA at the end */ + *ppos = SENTINEL_VMA_GATE; + return get_gate_vma(mm); } static void *m_next(struct seq_file *m, void *v, loff_t *ppos) { + struct proc_maps_private *priv = m->private; + struct vm_area_struct *vma; + if (*ppos == SENTINEL_VMA_GATE) { *ppos = SENTINEL_VMA_END; return NULL; } - return proc_get_vma(m, ppos); + vma = proc_get_vma(m, ppos); + if (vma) + return vma; + + /* Return gate VMA at the end */ + *ppos = SENTINEL_VMA_GATE; + return get_gate_vma(priv->lock_ctx.mm); } static void m_stop(struct seq_file *m, void *v) @@ -367,7 +311,12 @@ static void m_stop(struct seq_file *m, void *v) return; release_task_mempolicy(priv); - unlock_vma_range(&priv->lock_ctx); + if (priv->lock_ctx.mmap_locked) { + unlock_ctx_mm(&priv->lock_ctx); + } else { + unlock_ctx_vma(&priv->lock_ctx); + rcu_read_unlock(); + } mmput(mm); put_task_struct(priv->task); priv->task = NULL; @@ -560,23 +509,6 @@ static int pid_maps_open(struct inode *inode, struct file *file) PROCMAP_QUERY_VMA_FLAGS \ ) -#ifdef CONFIG_PER_VMA_LOCK - -static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) -{ - reset_lock_ctx(lock_ctx); - - return 0; -} - -static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) -{ - if (lock_ctx->mmap_locked) - unlock_ctx_mm(lock_ctx); - else - unlock_ctx_vma(lock_ctx); -} - static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, unsigned long addr) { @@ -612,26 +544,6 @@ static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ct return vma; } -#else /* CONFIG_PER_VMA_LOCK */ - -static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) -{ - return mmap_read_lock_killable(lock_ctx->mm); -} - -static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) -{ - mmap_read_unlock(lock_ctx->mm); -} - -static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, - unsigned long addr) -{ - return find_vma(lock_ctx->mm, addr); -} - -#endif /* CONFIG_PER_VMA_LOCK */ - static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx, unsigned long addr, u32 flags) { @@ -717,12 +629,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) if (!mm || !mmget_not_zero(mm)) return -ESRCH; - err = query_vma_setup(&lock_ctx); - if (err) { - mmput(mm); - return err; - } - + reset_lock_ctx(&lock_ctx); vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags); if (IS_ERR(vma)) { err = PTR_ERR(vma); @@ -796,7 +703,10 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) vm_file = get_file(vma->vm_file); /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ - query_vma_teardown(&lock_ctx); + if (lock_ctx.mmap_locked) + unlock_ctx_mm(&lock_ctx); + else + unlock_ctx_vma(&lock_ctx); mmput(mm); if (karg.build_id_size) { @@ -837,7 +747,10 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) return 0; out: - query_vma_teardown(&lock_ctx); + if (lock_ctx.mmap_locked) + unlock_ctx_mm(&lock_ctx); + else + unlock_ctx_vma(&lock_ctx); mmput(mm); out_file: if (vm_file) @@ -1314,8 +1227,6 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; -#ifdef CONFIG_PER_VMA_LOCK - static const struct mm_walk_ops smaps_walk_vma_lock_ops = { .pmd_entry = smaps_pte_range, .hugetlb_entry = smaps_hugetlb_range, @@ -1329,7 +1240,7 @@ static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, }; -static inline const struct mm_walk_ops * +static const struct mm_walk_ops * get_smaps_walk_ops(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) @@ -1337,7 +1248,7 @@ get_smaps_walk_ops(struct proc_maps_private *priv) return &smaps_walk_vma_lock_ops; } -static inline const struct mm_walk_ops * +static const struct mm_walk_ops * get_smaps_shmem_walk_ops(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) @@ -1345,36 +1256,26 @@ get_smaps_shmem_walk_ops(struct proc_maps_private *priv) return &smaps_shmem_walk_vma_lock_ops; } -#else /* CONFIG_PER_VMA_LOCK */ - -static inline const struct mm_walk_ops * -get_smaps_walk_ops(struct proc_maps_private *priv) -{ - return &smaps_walk_ops; -} - -static inline const struct mm_walk_ops * -get_smaps_shmem_walk_ops(struct proc_maps_private *priv) -{ - return &smaps_shmem_walk_ops; -} - -#endif /* CONFIG_PER_VMA_LOCK */ - -/* - * Gather mem stats from @vma with the indicated beginning - * address @start, and keep them in @mss. +/** + * smap_gather_stats_range() - Gather mem stats from a portion of the @vma. + * @priv: proc maps private state. + * @vma: The VMA to gather stats for. + * @mss: The accumulated stats. + * @start: The address from which to start. * - * Use vm_start of @vma as the beginning address if @start is 0. + * This gathers stats for the portion of the VMA starting at the @start + * address. */ -static void smap_gather_stats(struct proc_maps_private *priv, - struct vm_area_struct *vma, - struct mem_size_stats *mss, unsigned long start) +static void smap_gather_stats_range(struct proc_maps_private *priv, + struct vm_area_struct *vma, + struct mem_size_stats *mss, + unsigned long start) { const struct mm_walk_ops *ops = get_smaps_walk_ops(priv); + const bool is_partial = start > vma->vm_start; /* Invalid start */ - if (start >= vma->vm_end) + if (start < vma->vm_start || start >= vma->vm_end) return; if (vma == get_gate_vma(priv->lock_ctx.mm)) @@ -1385,33 +1286,39 @@ static void smap_gather_stats(struct proc_maps_private *priv, if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { /* - * For shared or readonly shmem mappings we know that all - * swapped out pages belong to the shmem object, and we can - * obtain the swap value much more efficiently. For private - * writable mappings, we might have COW pages that are - * not affected by the parent swapped out pages of the shmem - * object, so we have to distinguish them during the page walk. - * Unless we know that the shmem object (or the part mapped by - * our VMA) has no swapped out pages at all. + * CoW mappings might map anon folios that do not belong to + * shmem. Perform a less efficient page table walk in this + * situation, unless we know that the shmem object (or the + * part mapped by our VMA) has no swapped out pages at all. */ - unsigned long shmem_swapped = shmem_swap_usage(vma); + const unsigned long shmem_swapped = shmem_swap_usage(vma); + const bool is_cow = vma_is_cow_mapping(vma); - if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) || - !(vma->vm_flags & VM_WRITE))) { - mss->swap += shmem_swapped; - } else { + if (is_partial || (shmem_swapped && is_cow)) ops = get_smaps_shmem_walk_ops(priv); - } + else + mss->swap += shmem_swapped; } - if (!start) - walk_page_vma(vma, ops, mss); - else - walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); + walk_page_range_vma(vma, start, vma->vm_end, ops, mss); reacquire_rcu(priv); } +/** + * smap_gather_stats() - Gather mem stats from the entire @vma. + * @priv: proc maps private state. + * @vma: The VMA to gather stats for. + * @mss: The accumulated stats. + * + * This gathers stats for the whole of the VMA. + */ +static void smap_gather_stats(struct proc_maps_private *priv, + struct vm_area_struct *vma, struct mem_size_stats *mss) +{ + smap_gather_stats_range(priv, vma, mss, vma->vm_start); +} + #define SEQ_PUT_DEC(str, val) \ seq_put_decimal_ull_width(m, str, (val) >> 10, 8) @@ -1462,7 +1369,7 @@ static int show_smap(struct seq_file *m, void *v) struct vm_area_struct *vma = v; struct mem_size_stats mss = {}; - smap_gather_stats(priv, vma, &mss, 0); + smap_gather_stats(priv, vma, &mss); show_map_vma(m, vma); @@ -1487,12 +1394,14 @@ static int show_smap(struct seq_file *m, void *v) static int show_smaps_rollup(struct seq_file *m, void *v) { struct proc_maps_private *priv = m->private; + struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; + struct mm_struct *mm = lock_ctx->mm; struct mem_size_stats mss = {}; - struct mm_struct *mm = priv->lock_ctx.mm; + unsigned long last_vma_end = 0; + unsigned long vma_start = 0; struct vm_area_struct *vma; - unsigned long vma_start = 0, last_vma_end = 0; + loff_t pos = 0; int ret = 0; - VMA_ITERATOR(vmi, mm, 0); priv->task = get_proc_task(priv->inode); if (!priv->task) @@ -1503,89 +1412,63 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret = lock_ctx_mm(&priv->lock_ctx); - if (ret) - goto out_put_mm; - hold_task_mempolicy(priv); - vma = vma_next(&vmi); + rcu_read_lock(); + reset_lock_ctx(lock_ctx); + vma_iter_init(&priv->iter, mm, 0); + vma = proc_get_vma(m, &pos); if (unlikely(!vma)) goto empty_set; - vma_start = vma->vm_start; - do { - smap_gather_stats(priv, vma, &mss, 0); + if (!IS_ERR(vma)) + vma_start = vma->vm_start; + + while (vma) { + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto out_unlock; + } + + if (vma->vm_start < last_vma_end) { + /* + * After retaking the lock, already reported VMA grew + * or got merged with the next one and we found it + * again. Gather stats for the remaining portion by + * starting at last_vma_end. + */ + smap_gather_stats_range(priv, vma, &mss, last_vma_end); + } else { + /* Found next unreported VMA, start from its beginning */ + smap_gather_stats(priv, vma, &mss); + } last_vma_end = vma->vm_end; /* - * Release mmap_lock temporarily if someone wants to - * access it for write request. + * If the VMA lock is not taken, we hold the often contended + * mmap lock. This can happen if we had to fall back to the + * mmap lock. + * + * To relieve pressure, check if it is indeed contended, then + * temporarily release it. */ - if (mmap_lock_is_contended(mm)) { - vma_iter_invalidate(&vmi); - unlock_ctx_mm(&priv->lock_ctx); - ret = lock_ctx_mm(&priv->lock_ctx); - if (ret) { - release_task_mempolicy(priv); - goto out_put_mm; - } - + if (lock_ctx->mmap_locked && + mmap_lock_is_contended(lock_ctx->mm)) { + unlock_ctx_mm(lock_ctx); /* - * After dropping the lock, there are four cases to - * consider. See the following example for explanation. - * - * +------+------+-----------+ - * | VMA1 | VMA2 | VMA3 | - * +------+------+-----------+ - * | | | | - * 4k 8k 16k 400k - * - * Suppose we drop the lock after reading VMA2 due to - * contention, then we get: - * - * last_vma_end = 16k - * - * 1) VMA2 is freed, but VMA3 exists: - * - * vma_next(vmi) will return VMA3. - * In this case, just continue from VMA3. - * - * 2) VMA2 still exists: - * - * vma_next(vmi) will return VMA3. - * In this case, just continue from VMA3. - * - * 3) No more VMAs can be found: - * - * vma_next(vmi) will return NULL. - * No more things to do, just break. - * - * 4) (last_vma_end - 1) is the middle of a vma (VMA'): - * - * vma_next(vmi) will return VMA' whose range - * contains last_vma_end. - * Iterate VMA' from last_vma_end. + * Even though we previously fell back to mmap lock, + * we try taking VMA lock for the next VMA, since it + * might not be under modification. In the worst case + * we will fall back to mmap lock again. */ - vma = vma_next(&vmi); - /* Case 3 above */ - if (!vma) - break; - - /* Case 1 and 2 above */ - if (vma->vm_start >= last_vma_end) { - smap_gather_stats(priv, vma, &mss, 0); - last_vma_end = vma->vm_end; - continue; - } - - /* Case 4 above */ - if (vma->vm_end > last_vma_end) { - smap_gather_stats(priv, vma, &mss, last_vma_end); - last_vma_end = vma->vm_end; - } + rcu_read_lock(); + reset_lock_ctx(lock_ctx); + /* Resume from the last position. */ + pos = last_vma_end; + vma_iter_init(&priv->iter, mm, pos); } - } for_each_vma(vmi, vma); + vma = proc_get_vma(m, &pos); + } empty_set: show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); @@ -1594,10 +1477,14 @@ empty_set: __show_smap(m, &mss, true); +out_unlock: + if (lock_ctx->mmap_locked) { + unlock_ctx_mm(lock_ctx); + } else { + unlock_ctx_vma(lock_ctx); + rcu_read_unlock(); + } release_task_mempolicy(priv); - unlock_ctx_mm(&priv->lock_ctx); - -out_put_mm: mmput(mm); out_put_task: put_task_struct(priv->task); @@ -1687,7 +1574,7 @@ struct clear_refs_private { enum clear_refs_types type; }; -static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) +static bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { struct folio *folio; @@ -1703,8 +1590,8 @@ static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, return folio_maybe_dma_pinned(folio); } -static inline void clear_soft_dirty(struct vm_area_struct *vma, - unsigned long addr, pte_t *pte) +static void clear_soft_dirty(struct vm_area_struct *vma, unsigned long addr, + pte_t *pte) { if (!pgtable_supports_soft_dirty()) return; @@ -1735,7 +1622,7 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma, } #if defined(CONFIG_TRANSPARENT_HUGEPAGE) -static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, +static void clear_soft_dirty_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { pmd_t old, pmd = *pmdp; @@ -1761,7 +1648,7 @@ static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, } } #else -static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, +static void clear_soft_dirty_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { } @@ -1786,7 +1673,9 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, if (!pmd_present(*pmd)) goto out; - folio = pmd_folio(*pmd); + folio = vm_normal_folio_pmd(vma, addr, *pmd); + if (!folio) + goto out; /* Clear accessed and referenced bits. */ pmdp_test_and_clear_young(vma, addr, pmd); @@ -1961,7 +1850,7 @@ struct pagemapread { #define PM_END_OF_BUFFER 1 -static inline pagemap_entry_t make_pme(u64 frame, u64 flags) +static pagemap_entry_t make_pme(u64 frame, u64 flags) { return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags }; } @@ -2106,7 +1995,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, goto populate_pagemap; if (pmd_present(pmd)) { - page = pmd_page(pmd); + page = vm_normal_page_pmd(vma, addr, pmd); flags |= PM_PRESENT; if (pmd_soft_dirty(pmd)) @@ -3497,14 +3386,13 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; -#ifdef CONFIG_PER_VMA_LOCK static const struct mm_walk_ops show_numa_vma_lock_ops = { .hugetlb_entry = gather_hugetlb_stats, .pmd_entry = gather_pte_stats, .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, }; -static inline const struct mm_walk_ops * +static const struct mm_walk_ops * get_show_numa_ops(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) @@ -3512,16 +3400,6 @@ get_show_numa_ops(struct proc_maps_private *priv) return &show_numa_vma_lock_ops; } -#else /* CONFIG_PER_VMA_LOCK */ - -static inline const struct mm_walk_ops * -get_show_numa_ops(struct proc_maps_private *priv) -{ - return &show_numa_ops; -} - -#endif /* CONFIG_PER_VMA_LOCK */ - /* * Display pages allocated per node and memory policy via /proc. */ diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c index 406898247d7a..1fe70c4ecc54 100644 --- a/fs/proc/vmcore.c +++ b/fs/proc/vmcore.c @@ -474,29 +474,30 @@ static vm_fault_t mmap_vmcore_fault(struct vm_fault *vmf) pgoff_t index = vmf->pgoff; struct iov_iter iter; struct kvec kvec; - struct page *page; + struct folio *folio; loff_t offset; int rc; - page = find_or_create_page(mapping, index, GFP_KERNEL); - if (!page) + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) return VM_FAULT_OOM; - if (!PageUptodate(page)) { - offset = (loff_t) index << PAGE_SHIFT; - kvec.iov_base = page_address(page); - kvec.iov_len = PAGE_SIZE; - iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, PAGE_SIZE); + if (!folio_test_uptodate(folio)) { + offset = folio_pos(folio); + kvec.iov_base = folio_address(folio); + kvec.iov_len = folio_size(folio); + iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, folio_size(folio)); rc = __read_vmcore(&iter, &offset); if (rc < 0) { - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); return vmf_error(rc); } - SetPageUptodate(page); + folio_mark_uptodate(folio); } - unlock_page(page); - vmf->page = page; + folio_unlock(folio); + vmf->page = folio_file_page(folio, index); return 0; #else return VM_FAULT_SIGBUS; diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c index 244b835fc82a..99dd52715eee 100644 --- a/fs/ubifs/file.c +++ b/fs/ubifs/file.c @@ -12,14 +12,14 @@ * This file implements VFS file and inode operations for regular files, device * nodes and symlinks as well as address space operations. * - * UBIFS uses 2 page flags: @PG_private and @PG_checked. @PG_private is set if + * UBIFS uses folio->private and page flag @PG_checked. folio->private is set if * the page is dirty and is used for optimization purposes - dirty pages are - * not budgeted so the flag shows that 'ubifs_write_end()' should not release + * not budgeted so it shows that 'ubifs_write_end()' should not release * the budget for this page. The @PG_checked flag is set if full budgeting is * required for the page e.g., when it corresponds to a file hole or it is * beyond the file size. The budgeting is done in 'ubifs_write_begin()', because * it is OK to fail in this function, and the budget is released in - * 'ubifs_write_end()'. So the @PG_private and @PG_checked flags carry + * 'ubifs_write_end()'. So the folio->private and the @PG_checked flag carry * information about how the page was budgeted, to make it possible to release * the budget properly. * @@ -1509,7 +1509,7 @@ static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf) * * At the moment we do not know whether the folio is dirty or not, so we * assume that it is not and budget for a new folio. We could look at - * the @PG_private flag and figure this out, but we may race with write + * folio->private and figure this out, but we may race with write * back and the folio state may change by the time we lock it, so this * would need additional care. We do not bother with this at the * moment, although it might be good idea to do. Instead, we allocate diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index 7effebe297ea..2434c54a3570 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -2994,7 +2994,6 @@ struct xfs_btree_split_args { struct xfs_btree_cur **curp; int *stat; /* success/failure */ int result; - bool kswapd; /* allocation in kswapd context */ struct completion *done; struct work_struct work; }; @@ -3008,21 +3007,8 @@ xfs_btree_split_worker( { struct xfs_btree_split_args *args = container_of(work, struct xfs_btree_split_args, work); - unsigned long pflags; - unsigned long new_pflags = 0; unsigned int nofs_flags; - /* - * we are in a transaction context here, but may also be doing work - * in kswapd context, and hence we may need to inherit that state - * temporarily to ensure that we don't block waiting for memory reclaim - * in any way. - */ - if (args->kswapd) - new_pflags |= PF_MEMALLOC | PF_KSWAPD; - - current_set_flags_nested(&pflags, new_pflags); - /* * Don't use xfs_trans_set_context() here: it would overwrite the * caller's saved NOFS state in tp->t_pflags. Use a local scope. @@ -3033,14 +3019,12 @@ xfs_btree_split_worker( args->key, args->curp, args->stat); memalloc_nofs_restore(nofs_flags); - current_restore_flags_nested(&pflags, new_pflags); /* * Do not access args after complete() has run here. We don't own args * and the owner may run and free args before we return here. */ complete(args->done); - } /* @@ -3084,7 +3068,7 @@ xfs_btree_split( args.curp = curp; args.stat = stat; args.done = &done; - args.kswapd = current_is_kswapd(); + INIT_WORK_ONSTACK(&args.work, xfs_btree_split_worker); queue_work(xfs_alloc_wq, &args.work); wait_for_completion(&done); diff --git a/fs/xfs/xfs_platform.h b/fs/xfs/xfs_platform.h index 7b63ea26b186..981488778ccc 100644 --- a/fs/xfs/xfs_platform.h +++ b/fs/xfs/xfs_platform.h @@ -116,10 +116,6 @@ typedef __u32 xfs_nlink_t; #define xfs_blockgc_secs xfs_params.blockgc_timer.val #define current_cpu() (raw_smp_processor_id()) -#define current_set_flags_nested(sp, f) \ - (*(sp) = current->flags, current->flags |= (f)) -#define current_restore_flags_nested(sp, f) \ - (current->flags = ((current->flags & ~(f)) | (*(sp) & (f)))) #define NBBY 8 /* number of bits per byte */ diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index 89c21f84cffb..1cf739ee38aa 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -22,7 +22,6 @@ static inline int pgd_none(pgd_t pgd) { return 0; } static inline int pgd_bad(pgd_t pgd) { return 0; } static inline int pgd_present(pgd_t pgd) { return 1; } static inline void pgd_clear(pgd_t *pgd) { } -#define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd)) #define pgd_populate(mm, pgd, p4d) do { } while (0) #define pgd_populate_safe(mm, pgd, p4d) do { } while (0) diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index 36b6490ed180..ff4235cf84d7 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -33,7 +33,6 @@ static inline int pud_present(pud_t pud) { return 1; } static inline int pud_user(pud_t pud) { return 0; } static inline int pud_leaf(pud_t pud) { return 0; } static inline void pud_clear(pud_t *pud) { } -#define pmd_ERROR(pmd) (pud_ERROR((pmd).pud)) #define pud_populate(mm, pmd, pte) do { } while (0) diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index 356cbfbaab24..eedee8e3ad68 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -29,7 +29,6 @@ static inline int p4d_none(p4d_t p4d) { return 0; } static inline int p4d_bad(p4d_t p4d) { return 0; } static inline int p4d_present(p4d_t p4d) { return 1; } static inline void p4d_clear(p4d_t *p4d) { } -#define pud_ERROR(pud) (p4d_ERROR((pud).p4d)) #define p4d_populate(mm, p4d, pud) do { } while (0) #define p4d_populate_safe(mm, p4d, pud) do { } while (0) diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index bdcc2778ac64..9d827076db19 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -67,11 +67,8 @@ * - tlb_remove_table() * * tlb_remove_table() is the basic primitive to free page-table directories - * (__p*_free_tlb()). In it's most primitive form it is an alias for - * tlb_remove_page() below, for when page directories are pages and have no - * additional constraints. - * - * See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE. + * (__p*_free_tlb()). Page directories are freed after an RCU grace + * period - see the comment in mm/mmu_gather.c. * * - tlb_remove_page() / tlb_remove_page_size() * - __tlb_remove_folio_pages() / __tlb_remove_page_size() @@ -151,24 +148,15 @@ * This might be useful if your architecture has size specific TLB * invalidation instructions. * - * MMU_GATHER_TABLE_FREE + * Page directories (__p*_free_tlb()) are always freed via tlb_remove_table(), + * after an RCU grace period (see mm/mmu_gather.c). * - * This provides tlb_remove_table(), to be used instead of tlb_remove_page() - * for page directores (__p*_free_tlb()). + * This serialises against software page-table walkers, including architectures + * which do not use IPIs for remote TLB invalidates. * - * Useful if your architecture has non-page page directories. - * - * When used, an architecture is expected to provide __tlb_remove_table() or - * use the generic __tlb_remove_table(), which does the actual freeing of these - * pages. - * - * MMU_GATHER_RCU_TABLE_FREE - * - * Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see - * comment below). - * - * Useful if your architecture doesn't use IPIs for remote TLB invalidates - * and therefore doesn't naturally serialize with software page-table walkers. + * An architecture is expected to provide __tlb_remove_table() (see + * HAVE_ARCH_TLB_REMOVE_TABLE) or use the generic __tlb_remove_table(), which + * does the actual freeing of these pages. * * MMU_GATHER_NO_FLUSH_CACHE * @@ -200,12 +188,8 @@ * various ptep_get_and_clear() functions. */ -#ifdef CONFIG_MMU_GATHER_TABLE_FREE - struct mmu_table_batch { -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE struct rcu_head rcu; -#endif unsigned int nr; void *tables[]; }; @@ -224,23 +208,6 @@ static inline void __tlb_remove_table(void *table) extern void tlb_remove_table(struct mmu_gather *tlb, void *table); -#else /* !CONFIG_MMU_GATHER_TABLE_FREE */ - -static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page); -/* - * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based - * page directories and we can use the normal page batching to free them. - */ -static inline void tlb_remove_table(struct mmu_gather *tlb, void *table) -{ - struct ptdesc *ptdesc = (struct ptdesc *)table; - - pagetable_dtor(ptdesc); - tlb_remove_page(tlb, ptdesc_page(ptdesc)); -} -#endif /* CONFIG_MMU_GATHER_TABLE_FREE */ - -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE /* * This allows an architecture that does not use the linux page-tables for * hardware to skip the TLBI when freeing page tables. @@ -253,19 +220,6 @@ void tlb_remove_table_sync_one(void); void tlb_remove_table_sync_rcu(void); -#else - -#ifdef tlb_needs_table_invalidate -#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE -#endif - -static inline void tlb_remove_table_sync_one(void) { } - -static inline void tlb_remove_table_sync_rcu(void) { } - -#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ - - #ifndef CONFIG_MMU_GATHER_NO_GATHER /* * If we can't allocate a page to make a big batch of page pointers @@ -325,9 +279,7 @@ static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct struct mmu_gather { struct mm_struct *mm; -#ifdef CONFIG_MMU_GATHER_TABLE_FREE struct mmu_table_batch *batch; -#endif unsigned long start; unsigned long end; diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index e7a701ce029d..f8782b719026 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -201,12 +201,7 @@ static inline void kunmap_local_bh(const struct buffer_head *bh, void *addr) kunmap_local(addr); } -/* If we *know* page->private refers to buffer_heads */ -#define page_buffers(page) \ - ({ \ - BUG_ON(!PagePrivate(page)); \ - ((struct buffer_head *)page_private(page)); \ - }) +/* If we *know* folio->private refers to buffer_heads */ #define folio_buffers(folio) folio_get_private(folio) void buffer_check_dirty_writeback(struct folio *folio, @@ -223,7 +218,6 @@ void folio_set_bh(struct buffer_head *bh, struct folio *folio, unsigned long offset); struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, gfp_t gfp); -struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size); struct buffer_head *create_empty_buffers(struct folio *folio, unsigned long blocksize, unsigned long b_state); void end_buffer_read_sync(struct buffer_head *bh, int uptodate); diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 5dfa915a630e..2afb4cb2bb4f 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -154,6 +154,9 @@ struct cgroup *cgroup_get_from_path(const char *path); struct cgroup *cgroup_get_from_fd(int fd); struct cgroup *cgroup_v1v2_get_from_fd(int fd); +/* Was this controller blocked from v1 hierarchies by cgroup_no_v1= ? */ +bool cgroup1_ssid_disabled(int ssid); + int cgroup_attach_task_all(struct task_struct *from, struct task_struct *); int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from); diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 66a2f70e9e01..691c09f0a697 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -81,10 +81,6 @@ static inline unsigned long compact_gap(unsigned int order) return min(2UL << order, COMPACT_CLUSTER_MAX); } -static inline int current_is_kcompactd(void) -{ - return current->flags & PF_KCOMPACTD; -} #ifdef CONFIG_COMPACTION @@ -103,7 +99,7 @@ extern void compaction_defer_reset(struct zone *zone, int order, bool compaction_zonelist_suitable(struct alloc_context *ac, int order, int alloc_flags, gfp_t gfp_mask); - +bool current_is_kcompactd(void); extern void __meminit kcompactd_run(int nid); extern void __meminit kcompactd_stop(int nid); extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx); @@ -120,6 +116,11 @@ static inline bool compaction_suitable(struct zone *zone, int order, return false; } +static inline bool current_is_kcompactd(void) +{ + return false; +} + static inline void kcompactd_run(int nid) { } diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 70cb31d60538..0ea7150dba7b 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -292,6 +292,11 @@ static inline void *offset_to_ptr(const int *off) #define __ADDRESSABLE(sym) \ ___ADDRESSABLE(sym, __section(".discard.addressable")) +/* Enforce static storage duration. */ +#define ASSERT_STATIC_STORAGE(name) \ + static typeof(name) * const __always_unused \ + name##_storage_check = &(name) + /* * This returns a constant expression while determining if an argument is * a constant expression, most importantly without evaluating the argument. diff --git a/include/linux/damon.h b/include/linux/damon.h index 0c8b7ddef9ab..844b175120f0 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -153,6 +153,7 @@ enum damos_action { * @DAMOS_QUOTA_INACTIVE_MEM_BP: Inactive to total LRU memory ratio. * @DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: Scheme-eligible memory ratio of a * node in basis points (0-10000). + * @DAMOS_QUOTA_HUGEPAGE_MEM_BP: Huge page to total used memory ratio. * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. * * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. @@ -167,6 +168,7 @@ enum damos_quota_goal_metric { DAMOS_QUOTA_ACTIVE_MEM_BP, DAMOS_QUOTA_INACTIVE_MEM_BP, DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, + DAMOS_QUOTA_HUGEPAGE_MEM_BP, NR_DAMOS_QUOTA_GOAL_METRICS, }; @@ -357,7 +359,8 @@ struct damos_watermarks { * Total bytes that passed ops layer-handled DAMOS filters. * @qt_exceeds: Total number of times the quota of the scheme has exceeded. * @nr_snapshots: - * Total number of DAMON snapshots that the scheme has tried. + * Total number of DAMON snapshots that the scheme is completely + * tried to be applied. * * "Tried an action to a region" in this context means the DAMOS core logic * determined the region as eligible to apply the action. The access pattern @@ -396,14 +399,14 @@ struct damos_stat { * @DAMOS_FILTER_TYPE_UNMAPPED: Unmapped pages. * @DAMOS_FILTER_TYPE_ADDR: Address range. * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. - * @NR_DAMOS_FILTER_TYPES: Number of filter types. + * @DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: probe_hits weighted sum range. * - * All types except &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET - * are handled by the underlying &struct damon_operations as a part of scheme - * action trying, and therefore accounted as 'tried'. In contrast, - * &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET filters are handled - * by the core layer before trying of the action, and therefore not accounted - * as 'tried'. + * All types except &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET and + * &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM are handled by the underlying &struct + * damon_operations as a part of scheme action trying, and therefore accounted + * as 'tried'. In contrast, &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET + * and &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM filters are handled by the core layer + * before trying of the action, and therefore not accounted as 'tried'. * * Support for the operations-handled filters depends on the running * &struct damon_operations. @@ -417,7 +420,7 @@ enum damos_filter_type { DAMOS_FILTER_TYPE_UNMAPPED, DAMOS_FILTER_TYPE_ADDR, DAMOS_FILTER_TYPE_TARGET, - NR_DAMOS_FILTER_TYPES, + DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, }; /** @@ -431,6 +434,8 @@ enum damos_filter_type { * &damon_ctx->adaptive_targets if @type is * DAMOS_FILTER_TYPE_TARGET. * @sz_range: Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE. + * @range_min: Minimum value of range arguments. + * @range_max: Maximum value of range arguments. * * Before applying the &damos->action to a memory region, DAMOS checks if each * byte of the region matches to this given condition and avoid applying the @@ -447,6 +452,10 @@ struct damos_filter { struct damon_addr_range addr_range; int target_idx; struct damon_size_range sz_range; + struct { + unsigned long range_min; + unsigned long range_max; + }; }; /* private: */ /* List head for siblings. */ @@ -548,7 +557,7 @@ struct damos_migrate_dests { * * After applying the &action to each region, &stat is updated. * - * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots be same to or + * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots equals or is * greater than it, the scheme is deactivated. */ struct damos { @@ -627,6 +636,7 @@ enum damon_ops_id { * @update: Update operations-related data structures. * @prepare_access_checks: Prepare next access check of target regions. * @check_accesses: Check the accesses to target regions. + * @prep_probes: Prepare applying probes for each region. * @apply_probes: Apply probes for each region. * @get_scheme_score: Get the score of a region for a scheme. * @apply_scheme: Apply a DAMON-based operation scheme. @@ -654,6 +664,9 @@ enum damon_ops_id { * last preparation and update the number of observed accesses of each region. * It should also return max number of observed accesses that made as a result * of its update. The value will be used for regions adjustment threshold. + * @prep_probes should execute required &struct damon_prep for next &struct + * damon_probe applications to each region. It should also set + * &damon_region->sampling_addr of each region if ``set_samples`` is true. * @apply_probes should apply the data attribute probes to each region and * accordingly update the probe hits counter of the region. It should also * set &damon_region->sampling_addr of each region if ``set_samples`` is true. @@ -676,6 +689,7 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); + void (*prep_probes)(struct damon_ctx *context, bool set_samples); unsigned int (*apply_probes)(struct damon_ctx *context, bool set_samples, bool return_max_wsum); int (*get_scheme_score)(struct damon_ctx *context, @@ -739,15 +753,42 @@ struct damon_intervals_goal { unsigned long max_sample_us; }; +/** + * enum damon_prep_action - DAMON probing preparation action. + * + * @DAMON_PREP_SET_PGIDLE: Set the probing memory as idle page. + */ +enum damon_prep_action { + DAMON_PREP_SET_PGIDLE, +}; + +/** + * struct damon_prep - DAMON probing preparation request. + * + * @action: Action to do to the probing memory for the preparation. + */ +struct damon_prep { + enum damon_prep_action action; +/* private: */ + /* siblings list. */ + struct list_head list; +}; + /** * enum damon_filter_type - Type of &struct damon_filter * - * @DAMON_FILTER_TYPE_ANON: Anonymous pages. - * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. + * @DAMON_FILTER_TYPE_ANON: Anonymous pages. + * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. + * @DAMON_FILTER_TYPE_PGIDLE_UNSET: Pgidle is unset. + * @DAMON_FILTER_TYPE_PGIDLE_SET: Pgidle is set. + * @DAMON_FILTER_TYPE_HUGEPAGE_SIZE: Page is part of a hugepage. */ enum damon_filter_type { DAMON_FILTER_TYPE_ANON, DAMON_FILTER_TYPE_MEMCG, + DAMON_FILTER_TYPE_PGIDLE_UNSET, + DAMON_FILTER_TYPE_PGIDLE_SET, + DAMON_FILTER_TYPE_HUGEPAGE_SIZE, }; /** @@ -757,6 +798,8 @@ enum damon_filter_type { * @matching: Whether this filter is for the type-matching ones. * @allow: Whether the @type-@matching ones should pass this filter. * @memcg_id: Memcg id of the question if @type is DAMON_FILTER_MEMCG. + * @range_min: Minimum value of range arguments. + * @range_max: Maximum value of range arguments. */ struct damon_filter { enum damon_filter_type type; @@ -764,6 +807,10 @@ struct damon_filter { bool allow; union { u64 memcg_id; + struct { + unsigned long range_min; + unsigned long range_max; + }; }; /* private: */ /* Siblings list. */ @@ -778,6 +825,8 @@ struct damon_filter { struct damon_probe { unsigned int weight; /* private: */ + /* Preparation actions to apply to each probing memory. */ + struct list_head preps; /* Filters for assessing if a given region is for this probe. */ struct list_head filters; /* Siblings list. */ @@ -787,7 +836,8 @@ struct damon_probe { /** * struct damon_attrs - Monitoring attributes for accuracy/overhead control. * - * @sample_interval: The time between access samplings. + * @sample_interval: The time between access samplings. Zero is + * accepted. * @aggr_interval: The time between monitor results aggregations. * @ops_update_interval: The time between monitoring operations updates. * @intervals_goal: Intervals auto-tuning goal. @@ -957,6 +1007,12 @@ static inline unsigned long damon_sz_region(struct damon_region *r) return r->ar.end - r->ar.start; } +#define damon_for_each_prep(p, probe) \ + list_for_each_entry(p, &(probe)->preps, list) + +#define damon_for_each_prep_safe(p, next, probe) \ + list_for_each_entry_safe(p, next, &(probe)->preps, list) + #define damon_for_each_filter(f, p) \ list_for_each_entry(f, &(p)->filters, list) @@ -1010,6 +1066,9 @@ static inline unsigned long damon_sz_region(struct damon_region *r) #ifdef CONFIG_DAMON +struct damon_prep *damon_new_prep(enum damon_prep_action action); +void damon_add_prep(struct damon_probe *p, struct damon_prep *prep); + struct damon_filter *damon_new_filter(enum damon_filter_type type, bool matching, bool allow); void damon_add_filter(struct damon_probe *probe, struct damon_filter *f); @@ -1023,7 +1082,7 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r, struct damon_ctx *ctx); unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, struct damon_ctx *ctx); -unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, bool mv, struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, @@ -1054,7 +1113,7 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src); struct damon_target *damon_new_target(void); void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); -bool damon_targets_empty(struct damon_ctx *ctx); +int damon_set_target_pid(struct damon_target *t, int pid); void damon_free_target(struct damon_target *t); void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx); unsigned int damon_nr_regions(struct damon_target *t); @@ -1065,7 +1124,6 @@ int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs); void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, ssize_t nr_schemes); int damon_commit_ctx(struct damon_ctx *old_ctx, struct damon_ctx *new_ctx); -int damon_nr_running_ctxs(void); bool damon_is_registered_ops(enum damon_ops_id id); int damon_register_ops(struct damon_operations *ops); int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id); diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index b1cd143f3c42..c1331bf0b239 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -244,7 +244,7 @@ enum { * definitely preferable to use the flag rather than opencode endless * loop around allocator. * Allocating pages from the buddy with __GFP_NOFAIL and order > 1 is - * not supported. Please consider using kvmalloc() instead. + * discouraged. Please consider using kvmalloc() instead if possible. */ #define __GFP_IO ((__force gfp_t)___GFP_IO) #define __GFP_FS ((__force gfp_t)___GFP_FS) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 16c4c4caa126..80a5a03e9cee 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -171,7 +171,6 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio); extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; -extern struct list_head huge_boot_pages[MAX_NUMNODES]; void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); @@ -676,10 +675,6 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -#define HUGE_BOOTMEM_HVO 0x0001 -#define HUGE_BOOTMEM_ZONES_VALID 0x0002 -#define HUGE_BOOTMEM_CMA 0x0004 - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); @@ -699,7 +694,8 @@ enum hugetlb_alloc_flag { #define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) struct folio *hugetlb_alloc_folio(struct hstate *h, - struct mempolicy_interpreted *mpoli, u8 alloc_flags); + struct mempolicy_interpreted *mpoli, struct mm_struct *mm, + u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, diff --git a/include/linux/idr.h b/include/linux/idr.h index 789e23e67444..e2a4b6298511 100644 --- a/include/linux/idr.h +++ b/include/linux/idr.h @@ -16,6 +16,7 @@ #include #include #include +#include struct idr { struct radix_tree_root idr_rt; @@ -269,7 +270,9 @@ struct ida { #define IDA_INIT(name) { \ .xa = XARRAY_INIT(name, IDA_INIT_FLAGS) \ } -#define DEFINE_IDA(name) struct ida name = IDA_INIT(name) +#define DEFINE_IDA(name) \ + struct ida name = IDA_INIT(name); \ + ASSERT_STATIC_STORAGE(name) int ida_alloc_range(struct ida *, unsigned int min, unsigned int max, gfp_t); void ida_free(struct ida *, unsigned int id); diff --git a/include/linux/kernel-page-flags.h b/include/linux/kernel-page-flags.h index 196778a087c4..fe5ab6e50bd7 100644 --- a/include/linux/kernel-page-flags.h +++ b/include/linux/kernel-page-flags.h @@ -11,7 +11,6 @@ #define KPF_RESERVED 32 #define KPF_MLOCKED 33 #define KPF_OWNER_2 34 -#define KPF_PRIVATE 35 #define KPF_PRIVATE_2 36 #define KPF_OWNER_PRIVATE 37 #define KPF_ARCH 38 diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index f6491a92ead1..691f6012daea 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -9,6 +9,7 @@ */ #include +#include #include #include @@ -297,7 +298,8 @@ struct maple_tree { #endif #define DEFINE_MTREE(name) \ - struct maple_tree name = MTREE_INIT(name, 0) + struct maple_tree name = MTREE_INIT(name, 0); \ + ASSERT_STATIC_STORAGE(name) #define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) #define mtree_lock_nested(mas, subclass) \ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7d1c0ce189a8..64c183be8cbf 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -23,6 +23,7 @@ #include #include #include +#include struct mem_cgroup; struct obj_cgroup; @@ -66,11 +67,6 @@ struct mem_cgroup_reclaim_cookie { #define MEM_CGROUP_ID_SHIFT 16 -struct mem_cgroup_private_id { - int id; - refcount_t ref; -}; - struct memcg_vmstats_percpu; struct memcg1_events_percpu; struct memcg_vmstats; @@ -87,50 +83,48 @@ struct mem_cgroup_reclaim_iter { * per-node information in memory controller. */ struct mem_cgroup_per_node { - /* Keep the read-only fields at the start */ + /* Set when the memcg is created, then only read. */ + __cacheline_group_begin_aligned(memcg_pn_read_mostly); struct mem_cgroup *memcg; /* Back pointer, we cannot */ /* use container_of */ struct lruvec_stats_percpu __percpu *lruvec_stats_percpu; struct lruvec_stats *lruvec_stats; struct shrinker_info __rcu *shrinker_info; + struct obj_cgroup __rcu *objcg; + + __cacheline_group_end_aligned(memcg_pn_read_mostly); -#ifdef CONFIG_MEMCG_V1 /* - * Memcg-v1 only stuff in middle as buffer between read mostly fields - * and update often fields to avoid false sharing. If v1 stuff is - * not present, an explicit padding is needed. + * Keep lruvec on its own lines. Sharing them with lru_zone_size[] + * regressed, see commit f59adcf59332 ("mm: memcg: add cacheline + * padding after lruvec in mem_cgroup_per_node"). */ - - struct rb_node tree_node; /* RB tree node */ - unsigned long usage_in_excess;/* Set to the value by which */ - /* the soft limit is exceeded*/ - bool on_tree; -#else - CACHELINE_PADDING(_pad1_); -#endif - - /* Fields which get updated often at the end. */ + __cacheline_group_begin_aligned(memcg_pn_lruvec); struct lruvec lruvec; - CACHELINE_PADDING(_pad2_); - unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; + __cacheline_group_end_aligned(memcg_pn_lruvec); + + /* Written on every LRU update and on every reclaim iteration. */ + __cacheline_group_begin_aligned(memcg_pn_write_hot); + long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; - - /* - * objcg is wiped out as a part of the objcg repaprenting process. - * orig_objcg preserves a pointer (and a reference) to the original - * objcg until the end of live of memcg. - */ - struct obj_cgroup __rcu *objcg; - struct obj_cgroup *orig_objcg; - /* list of inherited objcgs, protected by objcg_lock */ - struct list_head objcg_list; - #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC /* slab stats for nmi context */ atomic_t slab_reclaimable; atomic_t slab_unreclaimable; #endif + __cacheline_group_end_aligned(memcg_pn_write_hot); + + /* Touched only when the memcg is reparented or freed. */ + __cacheline_group_begin_aligned(memcg_pn_cold); + /* + * orig_objcg preserves a pointer (and a reference) to the original + * objcg until the end of life of memcg. + */ + struct obj_cgroup *orig_objcg; + /* list of inherited objcgs, protected by objcg_lock */ + struct list_head objcg_list; + __cacheline_group_end_aligned(memcg_pn_cold); }; struct mem_cgroup_threshold { @@ -202,9 +196,6 @@ struct obj_cgroup { struct mem_cgroup { struct cgroup_subsys_state css; - /* Private memcg ID. Used to ID objects that outlive the cgroup */ - struct mem_cgroup_private_id id; - /* Accounted resources */ struct page_counter memory; /* Both v1 & v2 */ @@ -213,32 +204,53 @@ struct mem_cgroup { struct page_counter memsw; /* v1 only */ }; + /* Written on the charge, reclaim and socket paths. */ + __cacheline_group_begin_aligned(memcg_write_hot); + /* + * Hint of reclaim pressure for socket memory management. Note + * that this indicator should NOT be used in legacy cgroup mode + * where socket memory is accounted/charged separately. + */ + u64 socket_pressure; +#if BITS_PER_LONG < 64 + seqlock_t socket_pressure_seqlock; +#endif + /* + * memory.events is bumped for this memcg and all its ancestors, so a + * busy child dirties every ancestor. + */ + atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; + atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; + + /* vmpressure notifications. Written on every reclaim iteration. */ + struct vmpressure vmpressure; + + /* Written on every swap charge and uncharge. */ + refcount_t private_id_ref; + +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + /* MEMCG_KMEM for nmi context */ + atomic_t kmem_stat; +#endif + + /* Range enforcement for interrupt charges */ + struct irq_work high_irq_work; + struct work_struct high_work; + + __cacheline_group_end_aligned(memcg_write_hot); + + /* + * Off the charge and fault paths. Not write free: cgwb_domain is + * written on every writeout completion and mm_list on fork, exit and + * MGLRU aging. They are grouped here so those writes cannot land on + * a line that the fast paths read. + */ + __cacheline_group_begin_aligned(memcg_cold); /* registered local peak watchers */ struct list_head memory_peaks; struct list_head swap_peaks; spinlock_t peaks_lock; - /* Range enforcement for interrupt charges */ - struct work_struct high_work; - -#ifdef CONFIG_ZSWAP - unsigned long zswap_max; - - /* - * Prevent pages from this memcg from being written back from zswap to - * swap, and from being swapped out on zswap store failures. - */ - bool zswap_writeback; -#endif - - /* vmpressure notifications */ - struct vmpressure vmpressure; - - /* - * Should the OOM killer kill all belonging tasks, had it kill one? - */ - bool oom_group; - /* memory.events and memory.events.local */ struct cgroup_file events_file; struct cgroup_file events_local_file; @@ -246,37 +258,8 @@ struct mem_cgroup { /* handle for "memory.swap.events" */ struct cgroup_file swap_events_file; - /* memory.stat */ - struct memcg_vmstats *vmstats; - - /* memory.events */ - atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; - atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; - -#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC - /* MEMCG_KMEM for nmi context */ - atomic_t kmem_stat; -#endif - /* - * Hint of reclaim pressure for socket memroy management. Note - * that this indicator should NOT be used in legacy cgroup mode - * where socket memory is accounted/charged separately. - */ - u64 socket_pressure; -#if BITS_PER_LONG < 64 - seqlock_t socket_pressure_seqlock; -#endif - int kmemcg_id; - #ifdef CONFIG_CGROUP_WRITEBACK struct list_head cgwb_list; -#endif - - /* Keep the hot per-CPU stats pointer away from memory event counters. */ - struct memcg_vmstats_percpu __percpu *vmstats_percpu - ____cacheline_aligned_in_smp; - -#ifdef CONFIG_CGROUP_WRITEBACK struct wb_domain cgwb_domain; struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; #endif @@ -285,16 +268,17 @@ struct mem_cgroup { /* per-memcg mm_struct list */ struct lru_gen_mm_list mm_list; #endif + __cacheline_group_end_aligned(memcg_cold); #ifdef CONFIG_MEMCG_V1 + /* v1 only. Not grouped: v1 is legacy, sorting it is not worth it. */ + /* Legacy consumer-oriented counters */ struct page_counter kmem; /* v1 only */ struct page_counter tcpmem; /* v1 only */ struct memcg1_events_percpu __percpu *events_percpu; - unsigned long soft_limit; - /* protected by memcg_oom_lock */ bool oom_lock; int under_oom; @@ -325,6 +309,41 @@ struct mem_cgroup { int swappiness; #endif /* CONFIG_MEMCG_V1 */ + /* + * Set when the memcg is created and cleared when it is offlined. + * Never written on a hot path. + */ + __cacheline_group_begin_aligned(memcg_read_mostly); + /* Read on every stat update */ + struct memcg_vmstats_percpu __percpu *vmstats_percpu; + + /* memory.stat */ + struct memcg_vmstats *vmstats; + +#ifdef CONFIG_ZSWAP + unsigned long zswap_max; +#endif + + /* Private memcg ID. Used to ID objects that outlive the cgroup */ + int private_id; + + int kmemcg_id; + + /* + * Should the OOM killer kill all belonging tasks, had it kill one? + */ + bool oom_group; + +#ifdef CONFIG_ZSWAP + /* + * Prevent pages from this memcg from being written back from zswap to + * swap, and from being swapped out on zswap store failures. + */ + bool zswap_writeback; +#endif + /* Not padded: nodeinfo[] is read-mostly too, let it share the line. */ + __cacheline_group_end(memcg_read_mostly); + struct mem_cgroup_per_node *nodeinfo[]; }; @@ -377,10 +396,10 @@ enum objext_flags { * * The caller must ensure that the returned memcg won't be released. */ -static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg) { lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex)); - return READ_ONCE(objcg->memcg); + return objcg ? READ_ONCE(objcg->memcg) : NULL; } /* @@ -391,7 +410,7 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) * or NULL. This function assumes that the folio is known to have a * proper object cgroup pointer. */ -static inline struct obj_cgroup *folio_objcg(struct folio *folio) +static inline struct obj_cgroup *folio_objcg(const struct folio *folio) { unsigned long memcg_data = folio->memcg_data; @@ -429,11 +448,11 @@ static inline struct obj_cgroup *folio_objcg(struct folio *folio) * Note: The caller should hold an rcu read lock or cgroup_mutex to protect * memcg associated with a folio from being released. */ -static inline struct mem_cgroup *folio_memcg(struct folio *folio) +static inline struct mem_cgroup *folio_memcg(const struct folio *folio) { struct obj_cgroup *objcg = folio_objcg(folio); - return objcg ? obj_cgroup_memcg(objcg) : NULL; + return obj_cgroup_memcg(objcg); } /* @@ -442,7 +461,7 @@ static inline struct mem_cgroup *folio_memcg(struct folio *folio) * * Returns true if folio is charged to a memory cgroup, otherwise returns false. */ -static inline bool folio_memcg_charged(struct folio *folio) +static inline bool folio_memcg_charged(const struct folio *folio) { return folio->memcg_data != 0; } @@ -462,7 +481,7 @@ static inline bool folio_memcg_charged(struct folio *folio) * A caller should hold an rcu read lock to protect memcg associated with a * page from being released. */ -static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) +static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio) { /* * Because folio->memcg_data might be changed asynchronously @@ -476,17 +495,17 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); - return objcg ? obj_cgroup_memcg(objcg) : NULL; + return obj_cgroup_memcg(objcg); } -static inline struct mem_cgroup *page_memcg_check(struct page *page) +static inline struct mem_cgroup *page_memcg_check(const struct page *page) { if (PageTail(page)) return NULL; - return folio_memcg_check((struct folio *)page); + return folio_memcg_check((const struct folio *)page); } -static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg) { struct mem_cgroup *memcg; @@ -508,19 +527,19 @@ retry: * that the folio has an associated memory cgroup. It's not safe to call * this function against some types of folios, e.g. slab folios. */ -static inline bool folio_memcg_kmem(struct folio *folio) +static inline bool folio_memcg_kmem(const struct folio *folio) { VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio); return folio->memcg_data & MEMCG_DATA_KMEM; } -static inline bool PageMemcgKmem(struct page *page) +static inline bool PageMemcgKmem(const struct page *page) { return folio_memcg_kmem(page_folio(page)); } -static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) +static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg) { return (memcg == root_mem_cgroup); } @@ -536,7 +555,7 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) * and do not honour sc->memcg can use this to early-return 0 in per-memcg * contexts. */ -static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc) { return !sc->memcg || mem_cgroup_is_root(sc->memcg); } @@ -551,8 +570,8 @@ static inline bool mem_cgroup_disabled(void) return !cgroup_subsys_enabled(memory_cgrp_subsys); } -static inline void mem_cgroup_protection(struct mem_cgroup *root, - struct mem_cgroup *memcg, +static inline void mem_cgroup_protection(const struct mem_cgroup *root, + const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage) @@ -606,8 +625,8 @@ static inline void mem_cgroup_protection(struct mem_cgroup *root, void mem_cgroup_calculate_protection(struct mem_cgroup *root, struct mem_cgroup *memcg); -static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { /* * The root memcg doesn't account charges, and doesn't support @@ -618,8 +637,8 @@ static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, memcg == target; } -static inline bool mem_cgroup_below_low(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { if (mem_cgroup_unprotected(target, memcg)) return false; @@ -628,8 +647,8 @@ static inline bool mem_cgroup_below_low(struct mem_cgroup *target, page_counter_read(&memcg->memory); } -static inline bool mem_cgroup_below_min(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { if (mem_cgroup_unprotected(target, memcg)) return false; @@ -662,7 +681,8 @@ static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, return __mem_cgroup_charge(folio, mm, gfp); } -int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp); +int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, + gfp_t gfp); int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp); @@ -702,7 +722,7 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new); * @pgdat combination. This can be the node lruvec, if the memory * controller is disabled. */ -static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, +static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat) { struct mem_cgroup_per_node *mz; @@ -743,7 +763,7 @@ out: * their binding is stable if the returned lruvec matches the one the caller has * locked. Useful for lock batching. */ -static inline struct lruvec *folio_lruvec(struct folio *folio) +static inline struct lruvec *folio_lruvec(const struct folio *folio) { struct mem_cgroup *memcg = folio_memcg(folio); @@ -757,11 +777,11 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); struct mem_cgroup *get_mem_cgroup_from_current(void); -struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio); +struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio); -struct lruvec *folio_lruvec_lock(struct folio *folio); -struct lruvec *folio_lruvec_lock_irq(struct folio *folio); -struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +struct lruvec *folio_lruvec_lock(const struct folio *folio); +struct lruvec *folio_lruvec_lock_irq(const struct folio *folio); +struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flags); static inline @@ -820,16 +840,16 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, int (*)(struct task_struct *, void *), void *arg); -static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) +static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg) { if (mem_cgroup_disabled()) return 0; - return memcg->id.id; + return memcg->private_id; } struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id); -static inline u64 mem_cgroup_id(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg) { return memcg ? cgroup_id(memcg->css.cgroup) : 0; } @@ -841,14 +861,14 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) return mem_cgroup_from_css(seq_css(m)); } -static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) +static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec) { - struct mem_cgroup_per_node *mz; + const struct mem_cgroup_per_node *mz; if (mem_cgroup_disabled()) return NULL; - mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); return mz->memcg; } @@ -858,13 +878,13 @@ static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) * * Returns the parent memcg, or NULL if this is the root. */ -static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) +static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg) { return mem_cgroup_from_css(memcg->css.parent); } -static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, - struct mem_cgroup *root) +static inline bool mem_cgroup_is_descendant(const struct mem_cgroup *memcg, + const struct mem_cgroup *root) { if (root == memcg) return true; @@ -872,7 +892,7 @@ static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, } static inline bool mm_match_cgroup(struct mm_struct *mm, - struct mem_cgroup *memcg) + const struct mem_cgroup *memcg) { struct mem_cgroup *task_memcg; bool match = false; @@ -885,8 +905,8 @@ static inline bool mm_match_cgroup(struct mm_struct *mm, return match; } -struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio); -ino_t page_cgroup_ino(struct page *page); +struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio); +ino_t page_cgroup_ino(const struct page *page); static inline bool mem_cgroup_online(struct mem_cgroup *memcg) { @@ -899,13 +919,18 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages); static inline -unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, +unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx) { - struct mem_cgroup_per_node *mz; + long val; + const struct mem_cgroup_per_node *mz; - mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - return READ_ONCE(mz->lru_zone_size[zone_idx][lru]); + mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); + val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]); + if (WARN_ON_ONCE(val < 0)) + return 0; + + return val; } void __mem_cgroup_handle_over_high(gfp_t gfp_mask); @@ -916,9 +941,9 @@ static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask) __mem_cgroup_handle_over_high(gfp_mask); } -unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg); +unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg); -void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, +void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, struct task_struct *p); void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg); @@ -931,7 +956,7 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg); void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, int val); -static inline void mod_memcg_page_state(struct page *page, +static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val) { struct mem_cgroup *memcg; @@ -946,15 +971,16 @@ static inline void mod_memcg_page_state(struct page *page, rcu_read_unlock(); } -unsigned long memcg_events(struct mem_cgroup *memcg, int event); -unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx); -unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item); +unsigned long memcg_events(const struct mem_cgroup *memcg, int event); +unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx); +unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item); bool memcg_stat_item_valid(int idx); bool memcg_vm_event_item_valid(enum vm_event_item idx); -unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx); -unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +unsigned long lruvec_page_state(const struct lruvec *lruvec, + enum node_stat_item idx); +unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx); -unsigned long lruvec_page_state_local(struct lruvec *lruvec, +unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx); void mem_cgroup_flush_stats(struct mem_cgroup *memcg); @@ -965,7 +991,7 @@ void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val); void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count); -static inline void count_memcg_folio_events(struct folio *folio, +static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr) { struct mem_cgroup *memcg; @@ -1050,51 +1076,56 @@ void mem_cgroup_flush_workqueue(void); extern int mem_cgroup_init(void); #else /* CONFIG_MEMCG */ +static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg) +{ + return NULL; +} + #define MEM_CGROUP_ID_SHIFT 0 #define root_mem_cgroup (NULL) -static inline struct mem_cgroup *folio_memcg(struct folio *folio) +static inline struct mem_cgroup *folio_memcg(const struct folio *folio) { return NULL; } -static inline bool folio_memcg_charged(struct folio *folio) +static inline bool folio_memcg_charged(const struct folio *folio) { return false; } -static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) +static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio) { return NULL; } -static inline struct mem_cgroup *page_memcg_check(struct page *page) +static inline struct mem_cgroup *page_memcg_check(const struct page *page) { return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg) { return NULL; } -static inline bool folio_memcg_kmem(struct folio *folio) +static inline bool folio_memcg_kmem(const struct folio *folio) { return false; } -static inline bool PageMemcgKmem(struct page *page) +static inline bool PageMemcgKmem(const struct page *page) { return false; } -static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) +static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg) { return true; } -static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc) { return true; } @@ -1119,8 +1150,8 @@ static inline void memcg_memory_event_mm(struct mm_struct *mm, { } -static inline void mem_cgroup_protection(struct mem_cgroup *root, - struct mem_cgroup *memcg, +static inline void mem_cgroup_protection(const struct mem_cgroup *root, + const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage) @@ -1133,19 +1164,19 @@ static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root, { } -static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return true; } -static inline bool mem_cgroup_below_low(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return false; } -static inline bool mem_cgroup_below_min(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return false; } @@ -1156,9 +1187,10 @@ static inline int mem_cgroup_charge(struct folio *folio, return 0; } -static inline int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp) +static inline int mem_cgroup_charge_hugetlb(struct folio *folio, + struct mm_struct *mm, gfp_t gfp) { - return 0; + return 0; } static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, @@ -1184,25 +1216,25 @@ static inline void mem_cgroup_migrate(struct folio *old, struct folio *new) { } -static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, +static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat) { return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec(struct folio *folio) +static inline struct lruvec *folio_lruvec(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); return &pgdat->__lruvec; } -static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) +static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg) { return NULL; } static inline bool mm_match_cgroup(struct mm_struct *mm, - struct mem_cgroup *memcg) + const struct mem_cgroup *memcg) { return true; } @@ -1217,7 +1249,7 @@ static inline struct mem_cgroup *get_mem_cgroup_from_current(void) return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio) +static inline struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio) { return NULL; } @@ -1250,7 +1282,7 @@ static inline void mem_cgroup_put(struct mem_cgroup *memcg) { } -static inline struct lruvec *folio_lruvec_lock(struct folio *folio) +static inline struct lruvec *folio_lruvec_lock(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1259,7 +1291,7 @@ static inline struct lruvec *folio_lruvec_lock(struct folio *folio) return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) +static inline struct lruvec *folio_lruvec_lock_irq(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1268,7 +1300,7 @@ static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +static inline struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flagsp) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1296,7 +1328,7 @@ static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, { } -static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) +static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg) { return 0; } @@ -1308,7 +1340,7 @@ static inline struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id) return NULL; } -static inline u64 mem_cgroup_id(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg) { return 0; } @@ -1323,7 +1355,7 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) return NULL; } -static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) +static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec) { return NULL; } @@ -1334,19 +1366,20 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg) } static inline -unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, +unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx) { return 0; } -static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) +static inline unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg) { return 0; } static inline void -mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) +mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, + struct task_struct *p) { } @@ -1375,17 +1408,17 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg, { } -static inline void mod_memcg_page_state(struct page *page, +static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val) { } -static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) +static inline unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx) { return 0; } -static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item) +static inline unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item) { return 0; } @@ -1400,19 +1433,19 @@ static inline bool memcg_vm_event_item_valid(enum vm_event_item idx) return false; } -static inline unsigned long lruvec_page_state(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state(lruvec_pgdat(lruvec), idx); } -static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state_monotonic(lruvec_pgdat(lruvec), idx); } -static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state(lruvec_pgdat(lruvec), idx); @@ -1440,7 +1473,7 @@ static inline void count_memcg_events(struct mem_cgroup *memcg, { } -static inline void count_memcg_folio_events(struct folio *folio, +static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr) { } @@ -1474,7 +1507,7 @@ static inline void mem_cgroup_flush_workqueue(void) { } static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ -static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) +static inline struct lruvec *parent_lruvec(const struct lruvec *lruvec) { struct mem_cgroup *memcg; @@ -1537,15 +1570,15 @@ static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long } /* Test requires a stable folio->memcg binding, see folio_memcg() */ -static inline bool folio_matches_lruvec(struct folio *folio, - struct lruvec *lruvec) +static inline bool folio_matches_lruvec(const struct folio *folio, + const struct lruvec *lruvec) { return lruvec_pgdat(lruvec) == folio_pgdat(folio) && lruvec_memcg(lruvec) == folio_memcg(folio); } /* Don't lock again iff page's lruvec locked */ -static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, +static inline struct lruvec *folio_lruvec_relock_irq(const struct folio *folio, struct lruvec *locked_lruvec) { if (locked_lruvec) { @@ -1559,7 +1592,7 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, } /* Don't lock again iff folio's lruvec locked */ -static inline void folio_lruvec_relock_irqsave(struct folio *folio, +static inline void folio_lruvec_relock_irqsave(const struct folio *folio, struct lruvec **lruvecp, unsigned long *flags) { if (*lruvecp) { @@ -1651,7 +1684,7 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg) write_sequnlock_irqrestore(&memcg->socket_pressure_seqlock, flags); } -static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg) { unsigned int seq; u64 val; @@ -1669,7 +1702,7 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg) WRITE_ONCE(memcg->socket_pressure, jiffies + HZ); } -static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg) { return READ_ONCE(memcg->socket_pressure); } @@ -1736,7 +1769,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order); * needs to be used outside of the local scope. */ struct obj_cgroup *current_obj_cgroup(void); -struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio); +struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio); static inline struct obj_cgroup *get_obj_cgroup_from_current(void) { @@ -1782,7 +1815,7 @@ static inline void memcg_kmem_uncharge_page(struct page *page, int order) * A helper for accessing memcg's kmem_id, used for getting * corresponding LRU lists. */ -static inline int memcg_kmem_id(struct mem_cgroup *memcg) +static inline int memcg_kmem_id(const struct mem_cgroup *memcg) { return memcg ? memcg->kmemcg_id : -1; } @@ -1839,7 +1872,7 @@ static inline void __memcg_kmem_uncharge_page(struct page *page, int order) { } -static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio) +static inline struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio) { return NULL; } @@ -1854,7 +1887,7 @@ static inline bool memcg_kmem_online(void) return false; } -static inline int memcg_kmem_id(struct mem_cgroup *memcg) +static inline int memcg_kmem_id(const struct mem_cgroup *memcg) { return -1; } @@ -1870,7 +1903,7 @@ static inline void count_objcg_events(struct obj_cgroup *objcg, { } -static inline ino_t page_cgroup_ino(struct page *page) +static inline ino_t page_cgroup_ino(const struct page *page) { return 0; } @@ -1890,11 +1923,21 @@ static inline bool memcg_is_dying(struct mem_cgroup *memcg) } #endif /* CONFIG_MEMCG */ +#if defined(CONFIG_MEMCG) && defined(CONFIG_LRU_GEN) +void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg); +#else +static inline void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg) +{ +} +#endif + #if defined(CONFIG_MEMCG) && defined(CONFIG_ZSWAP) bool obj_cgroup_may_zswap(struct obj_cgroup *objcg); void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size); void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size); -bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg); +bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg); #else static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) { @@ -1908,7 +1951,7 @@ static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) { } -static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) +static inline bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg) { /* if zswap is disabled, do not block pages going to the swapping device */ return true; @@ -1919,13 +1962,9 @@ static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) /* Cgroup v1-related declarations */ #ifdef CONFIG_MEMCG_V1 -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned); - bool mem_cgroup_oom_synchronize(bool wait); -static inline bool task_in_memcg_oom(struct task_struct *p) +static inline bool task_in_memcg_oom(const struct task_struct *p) { return p->memcg_in_oom; } @@ -1943,15 +1982,7 @@ static inline void mem_cgroup_exit_user_fault(void) } #else /* CONFIG_MEMCG_V1 */ -static inline -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned) -{ - return 0; -} - -static inline bool task_in_memcg_oom(struct task_struct *p) +static inline bool task_in_memcg_oom(const struct task_struct *p) { return false; } diff --git a/include/linux/mm.h b/include/linux/mm.h index dd09c438fa23..0a2a7fc4a442 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -38,6 +38,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -57,16 +58,6 @@ static inline unsigned long totalram_pages(void) return (unsigned long)atomic_long_read(&_totalram_pages); } -static inline void totalram_pages_inc(void) -{ - atomic_long_inc(&_totalram_pages); -} - -static inline void totalram_pages_dec(void) -{ - atomic_long_dec(&_totalram_pages); -} - static inline void totalram_pages_add(long count) { atomic_long_add(count, &_totalram_pages); @@ -928,7 +919,6 @@ static inline void vma_numab_state_free(struct vm_area_struct *vma) {} * These must be here rather than mmap_lock.h as dependent on vm_fault type, * declared in this header. */ -#ifdef CONFIG_PER_VMA_LOCK static inline void release_fault_lock(struct vm_fault *vmf) { if (vmf->flags & FAULT_FLAG_VMA_LOCK) @@ -944,17 +934,6 @@ static inline void assert_fault_locked(const struct vm_fault *vmf) else mmap_assert_locked(vmf->vma->vm_mm); } -#else -static inline void release_fault_lock(struct vm_fault *vmf) -{ - mmap_read_unlock(vmf->vma->vm_mm); -} - -static inline void assert_fault_locked(const struct vm_fault *vmf) -{ - mmap_assert_locked(vmf->vma->vm_mm); -} -#endif /* CONFIG_PER_VMA_LOCK */ static inline bool mm_flags_test(int flag, const struct mm_struct *mm) { @@ -1551,11 +1530,6 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) vma->vm_ops = NULL; } -static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) -{ - desc->vm_ops = NULL; -} - static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; @@ -2075,20 +2049,21 @@ vm_fault_t finish_fault(struct vm_fault *vmf); * * A pagecache page contains an opaque `private' member, which belongs to the * page's address_space. Usually, this is the address of a circular list of - * the page's disk buffers. PG_private must be set to tell the VM to call - * into the filesystem to release these pages. + * the page's disk buffers. It tells the VM to call into the filesystem to + * release these pages. * * A folio may belong to an inode's memory mapping. In this case, * folio->mapping points to the inode, and folio->index is the file * offset of the folio, in units of PAGE_SIZE. * - * If pagecache pages are not associated with an inode, they are said to be - * anonymous pages. These may become associated with the swapcache, and in that - * case PG_swapcache is set, and page->private is an offset into the swapcache. + * If pagecache folios are not associated with an inode, they are said to be + * anonymous folios. These may become associated with the swapcache, and in that + * case PG_swapcache is set, and folio->private is an offset into the swapcache. * * In either case (swapcache or inode backed), the pagecache itself holds one - * reference to the page. Setting PG_private should also increment the - * refcount. The each user mapping also has a reference to the page. + * reference to the folio. Attaching filesystem private data via + * folio_attach_private() also increments the refcount. Each user mapping also + * has a reference to the folio. * * The pagecache pages are stored in a per-mapping radix tree, which is * rooted at mapping->i_pages, and indexed by offset. @@ -2644,12 +2619,23 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ + set_page_section(page, pfn_to_section_nr(pfn)); +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; @@ -2872,9 +2858,7 @@ static inline void set_page_links(struct page *page, enum zone_type zone, { set_page_zone(page, zone); set_page_node(page, node); -#ifdef SECTION_IN_PAGE_FLAGS - set_page_section(page, pfn_to_section_nr(pfn)); -#endif + set_page_section_from_pfn(page, pfn); } /** @@ -3022,9 +3006,9 @@ static inline bool folio_maybe_mapped_shared(struct folio *folio) * @folio: the folio * * Calculate the expected folio refcount, taking references from the pagecache, - * swapcache, PG_private and page table mappings into account. Useful in - * combination with folio_ref_count() to detect unexpected references (e.g., - * GUP or other temporary references). + * swapcache, private data (folio->private != NULL) and page table mappings into + * account. Useful in combination with folio_ref_count() to detect unexpected + * references (e.g., GUP or other temporary references). * * Does currently not consider references from the LRU cache. If the folio * was isolated from the LRU (which is the case during migration or split), @@ -3062,10 +3046,15 @@ static inline int folio_expected_ref_count(const struct folio *folio) ref_count += folio_test_swapcache(folio) << order; if (!folio_test_anon(folio)) { - /* One reference per page from the pagecache. */ - ref_count += !!folio->mapping << order; - /* One reference from PG_private. */ - ref_count += folio_test_private(folio); + /* + * One reference per page from the pagecache. + * Use data_race() since folio might not be locked. + */ + ref_count += !!data_race(folio->mapping) << order; + /* + * One reference from filesystem private data. + */ + ref_count += folio_has_attached_private(folio); } /* One reference per page table mapping. */ @@ -4083,12 +4072,6 @@ static inline void free_reserved_page(struct page *page) free_reserved_pages(page, 0); } -static inline void mark_page_reserved(struct page *page) -{ - SetPageReserved(page); - adjust_managed_page_count(page, -1); -} - static inline void free_reserved_ptdesc(struct ptdesc *pt) { free_reserved_page(ptdesc_page(pt)); @@ -4408,9 +4391,8 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the * page offset within the file. * - * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies - * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap - * regions have their page offset set to the first PFN in the range. + * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions + * have their page offset set to the first PFN in the range. * * Returns: The page offset of the start of @vma. */ @@ -5140,7 +5122,6 @@ static inline void print_vma_addr(char *prefix, unsigned long rip) } #endif -unsigned long section_map_size(void); struct page * __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); @@ -5159,9 +5140,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); -int vmemmap_populate_hvo(unsigned long start, unsigned long end, - unsigned int order, struct zone *zone, - unsigned long headsize); void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize); void vmemmap_populate_print_last(void); @@ -5196,7 +5174,6 @@ static inline void vmem_altmap_free(struct vmem_altmap *altmap, } #endif -#define VMEMMAP_RESERVE_NR 2 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) @@ -5204,6 +5181,9 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, unsigned long nr_pages; unsigned long nr_vmemmap_pages; + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) + return false; + if (!pgmap || !is_power_of_2(sizeof(struct page))) return false; @@ -5213,7 +5193,7 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, * For vmemmap optimization with DAX we need minimum 2 vmemmap * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst */ - return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR); + return !altmap && (nr_vmemmap_pages > VMEMMAP_OPTIMIZATION_PAGES); } /* * If we don't have an architecture override, use the generic rule diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 621c8653d8f7..ab69b9930893 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -30,6 +30,17 @@ static inline int folio_is_file_lru(const struct folio *folio) return !folio_test_swapbacked(folio); } +/** + * folio_flags_is_file_lru - Should the folio be on a file LRU or anon LRU? + * @flags: The folio's flags. + * + * Just like folio_is_file_lru but take the folio flags directly instead. + */ +static inline int folio_flags_is_file_lru(const unsigned long *flags) +{ + return !test_bit(PG_swapbacked, flags); +} + static __always_inline void __update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, long nr_pages) @@ -142,10 +153,66 @@ static inline int lru_tier_from_refs(int refs, bool workingset) return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs); } -static inline int folio_lru_refs(const struct folio *folio) +/** + * lru_set_gen_flags - Set the LRU generation number to specified folio flags. + * @flags: pointer to the folio flags + * @gen: generation number, between 0 and (MAX_NR_GENS - 1), inclusive. + */ +static inline void lru_set_gen_flags(unsigned long *flags, int gen) { - unsigned long flags = READ_ONCE(folio->flags.f); + BUILD_BUG_ON(LRU_GEN_MASK & LRU_REFS_MASK); + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS || gen < 0); + /* Store gen offset by 1, zero means the folio is off-list. */ + *flags &= ~LRU_GEN_MASK; + *flags |= (gen + 1UL) << LRU_GEN_PGOFF; +} +/** + * lru_get_gen_flags - Return the LRU generation number from folio flags. + * @flags: folio flags + * + * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns + * -1 if the flags indicate the folio is off the list (e.g., isolated). + */ +static inline int lru_get_gen_flags(unsigned long flags) +{ + int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + + /* Exclude the legal -1 from the unsigned MAX_NR_GENS comparison */ + VM_WARN_ON_ONCE(gen != -1 && gen >= MAX_NR_GENS); + return gen; +} + +/** + * lru_set_refs_flags - Set the LRU referenced count to folio flags. + * @flags: pointer to the folio flags + * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive. + * + * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the + * remaining refs. For classical LRU the extra bits are not used, so it can + * also be seen as the refs count never exceeds 1. In both cases, refs == 1 + * means PG_referenced is set and the extra bits are zero, and refs == 0 means + * PG_referenced and the extra bits are all unset. + */ +static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs) +{ + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); + BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1); + + *flags &= ~LRU_REFS_FLAGS; + if (!refs) + return; + *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF)); +} + +/** + * lru_get_refs_flags - Return LRU referenced / access count from folio flags. + * @flags: folio flags + * + * Reads the LRU referenced count set by lru_set_refs_flags(). + */ +static inline int lru_get_refs_flags(unsigned long flags) +{ if (!(flags & BIT(PG_referenced))) return 0; /* @@ -155,11 +222,24 @@ static inline int folio_lru_refs(const struct folio *folio) return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1; } +static inline int folio_lru_refs(const struct folio *folio) +{ + return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0))); +} + +static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs) +{ + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + + do { + new_flags = old_flags; + lru_set_refs_flags(&new_flags, refs); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); +} + static inline int folio_lru_gen(const struct folio *folio) { - unsigned long flags = READ_ONCE(folio->flags.f); - - return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0))); } static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen) @@ -270,7 +350,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, gen = lru_gen_from_seq(seq); flags = (gen + 1UL) << LRU_GEN_PGOFF; /* see the comment on MIN_NR_GENS about PG_active */ - set_mask_bits(&folio->flags.f, LRU_GEN_MASK | BIT(PG_active), flags); + set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags); lru_gen_update_size(lruvec, folio, -1, gen); /* for folio_rotate_reclaimable() */ @@ -295,7 +375,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, /* for folio_migrate_flags() */ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; - flags = set_mask_bits(&folio->flags.f, LRU_GEN_MASK, flags); + flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags); gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; lru_gen_update_size(lruvec, folio, gen, -1); @@ -304,11 +384,19 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return true; } -static inline void folio_migrate_refs(struct folio *new, const struct folio *old) +/** + * folio_migrate_lru_refs - copy the reference state to a new folio + * @new: the destination folio + * @old: the source folio + * + * Transfer the reference state to @new during migration: the MGLRU + * refs count, including PG_referenced, or just PG_referenced for the + * active/inactive LRU. + */ +static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { - unsigned long refs = READ_ONCE(old->flags.f) & LRU_REFS_MASK; - - set_mask_bits(&new->flags.f, LRU_REFS_MASK, refs); + BUILD_BUG_ON(LRU_REFS_MASK & BIT(PG_referenced)); + folio_set_lru_refs(new, folio_lru_refs(old)); } #else /* !CONFIG_LRU_GEN */ @@ -337,9 +425,10 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return false; } -static inline void folio_migrate_refs(struct folio *new, const struct folio *old) +static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { - + if (folio_test_referenced(old)) + folio_set_referenced(new); } #endif /* CONFIG_LRU_GEN */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index f3e5a2fadbe5..0720a4e98286 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -108,7 +108,7 @@ struct page { }; /** * @private: Mapping-private opaque data. - * Usually used for buffer_heads if PagePrivate. + * Usually used for buffer_heads. * Used for swp_entry_t if swapcache flag set. * Indicates order in the buddy system if PageBuddy * or on pcp_llist. @@ -675,7 +675,7 @@ static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc) #define STRUCT_PAGE_MAX_SHIFT (order_base_2(sizeof(struct page))) /* - * page_private can be used on tail pages. However, PagePrivate is only + * page_private can be used on tail pages. However, it is only * checked by the VM on the head page. So page_private on the tail pages * should be used for data that's ancillary to the head page (eg attaching * buffer heads to tail pages after attaching buffer heads to the head page) @@ -950,7 +950,6 @@ struct vm_area_struct { vma_flags_t flags; }; -#ifdef CONFIG_PER_VMA_LOCK /* * Can only be written (using WRITE_ONCE()) while holding both: * - mmap_lock (in write mode) @@ -966,7 +965,7 @@ struct vm_area_struct { * slowpath. */ unsigned int vm_lock_seq; -#endif + /* * Low 32-bits of anonymous page offset. * See vma_start_anon_pgoff() comment for details. @@ -1003,7 +1002,6 @@ struct vm_area_struct { #ifdef CONFIG_NUMA_BALANCING struct vma_numab_state *numab_state; /* NUMA Balancing state */ #endif -#ifdef CONFIG_PER_VMA_LOCK /* * Used to keep track of firstly, whether the VMA is attached, secondly, * if attached, how many read locks are taken, and thirdly, if the @@ -1046,7 +1044,6 @@ struct vm_area_struct { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map vmlock_dep_map; #endif -#endif #ifdef CONFIG_64BIT /* * High 32-bits of anonymous page offset. @@ -1254,7 +1251,6 @@ struct mm_struct { * init_mm.mmlist, and are protected * by mmlist_lock */ -#ifdef CONFIG_PER_VMA_LOCK struct rcuwait vma_writer_wait; /* * This field has lock-like semantics, meaning it is sometimes @@ -1274,7 +1270,7 @@ struct mm_struct { * mmap_lock. */ seqcount_t mm_lock_seq; -#endif + struct futex_mm_data futex; unsigned long hiwater_rss; /* High-watermark of RSS usage */ diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index b8a13b8d36a4..28e3696ce9ff 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -76,8 +76,6 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) rwsem_assert_held_write(&mm->mmap_lock); } -#ifdef CONFIG_PER_VMA_LOCK - #ifdef CONFIG_LOCKDEP #define __vma_lockdep_map(vma) (&vma->vmlock_dep_map) #else @@ -230,10 +228,14 @@ static inline void vma_refcount_put(struct vm_area_struct *vma) } /* - * Use only while holding mmap read lock which guarantees that locking will not - * fail (nobody can concurrently write-lock the vma). vma_start_read() should + * Use only while holding mmap read lock which guarantees that vma lock is not + * contended (nobody can concurrently write-lock the vma). vma_start_read() should * not be used in such cases because it might fail due to mm_lock_seq overflow. * This functionality is used to obtain vma read lock and drop the mmap read lock. + * + * VMA can't be detached while we are holding mmap lock, therefore in practice this + * function can fail only when there are so many readers that vm_refcnt overflows. + * The failure case is very unlikely and is already annotated as such internally. */ static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass) { @@ -249,16 +251,23 @@ static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int } /* - * Use only while holding mmap read lock which guarantees that locking will not - * fail (nobody can concurrently write-lock the vma). vma_start_read() should + * Use only while holding mmap read lock which guarantees that vma lock is not + * contended (nobody can concurrently write-lock the vma). vma_start_read() should * not be used in such cases because it might fail due to mm_lock_seq overflow. * This functionality is used to obtain vma read lock and drop the mmap read lock. + * + * VMA can't be detached while we are holding mmap lock, therefore in practice this + * function can fail only when there are so many readers that vm_refcnt overflows. + * The failure case is very unlikely and is already annotated as such internally. */ static inline bool vma_start_read_locked(struct vm_area_struct *vma) { return vma_start_read_locked_nested(vma, 0); } +struct vm_area_struct *vma_start_read_unlocked(struct mm_struct *mm, + unsigned long address); + static inline void vma_end_read(struct vm_area_struct *vma) { vma_refcount_put(vma); @@ -297,6 +306,9 @@ int __vma_start_write(struct vm_area_struct *vma, int state); */ static inline void vma_start_write(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + if (__is_vma_write_locked(vma)) return; @@ -319,6 +331,9 @@ static inline void vma_start_write(struct vm_area_struct *vma) static inline __must_check int vma_start_write_killable(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return 0; + if (__is_vma_write_locked(vma)) return 0; @@ -331,6 +346,11 @@ int vma_start_write_killable(struct vm_area_struct *vma) */ static inline void vma_assert_write_locked(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) { + mmap_assert_write_locked(vma->vm_mm); + return; + } + VM_WARN_ON_ONCE_VMA(!__is_vma_write_locked(vma), vma); } @@ -343,6 +363,11 @@ static inline void vma_assert_locked(struct vm_area_struct *vma) { unsigned int refcnt; + if (!IS_ENABLED(CONFIG_MMU)) { + mmap_assert_locked(vma->vm_mm); + return; + } + if (IS_ENABLED(CONFIG_LOCKDEP)) { if (!lock_is_held(__vma_lockdep_map(vma))) vma_assert_write_locked(vma); @@ -432,6 +457,9 @@ static inline bool vma_is_attached(struct vm_area_struct *vma) */ static inline void vma_assert_attached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + WARN_ON_ONCE(!vma_is_attached(vma)); } @@ -442,6 +470,9 @@ static inline void vma_assert_detached(struct vm_area_struct *vma) static inline void vma_mark_attached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + vma_assert_write_locked(vma); vma_assert_detached(vma); refcount_set_release(&vma->vm_refcnt, 1); @@ -451,6 +482,9 @@ void __vma_exclude_readers_for_detach(struct vm_area_struct *vma); static inline void vma_mark_detached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + vma_assert_write_locked(vma); vma_assert_attached(vma); @@ -484,54 +518,6 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm, struct vma_iterator *iter, unsigned long address); -#else /* CONFIG_PER_VMA_LOCK */ - -static inline void mm_lock_seqcount_init(struct mm_struct *mm) {} -static inline void mm_lock_seqcount_begin(struct mm_struct *mm) {} -static inline void mm_lock_seqcount_end(struct mm_struct *mm) {} - -static inline bool mmap_lock_speculate_try_begin(struct mm_struct *mm, unsigned int *seq) -{ - return false; -} - -static inline bool mmap_lock_speculate_retry(struct mm_struct *mm, unsigned int seq) -{ - return true; -} -static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) {} -static inline void vma_end_read(struct vm_area_struct *vma) {} -static inline void vma_start_write(struct vm_area_struct *vma) {} -static inline __must_check -int vma_start_write_killable(struct vm_area_struct *vma) { return 0; } -static inline void vma_assert_write_locked(struct vm_area_struct *vma) - { mmap_assert_write_locked(vma->vm_mm); } -static inline bool vma_is_attached(struct vm_area_struct *vma) - { return true; } -static inline void vma_assert_attached(struct vm_area_struct *vma) {} -static inline void vma_assert_detached(struct vm_area_struct *vma) {} -static inline void vma_mark_attached(struct vm_area_struct *vma) {} -static inline void vma_mark_detached(struct vm_area_struct *vma) {} - -static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, - unsigned long address) -{ - return NULL; -} - -static inline void vma_assert_locked(struct vm_area_struct *vma) -{ - mmap_assert_locked(vma->vm_mm); -} - -static inline void vma_assert_stabilised(struct vm_area_struct *vma) -{ - /* If no VMA locks, then either mmap lock suffices to stabilise. */ - mmap_assert_locked(vma->vm_mm); -} - -#endif /* CONFIG_PER_VMA_LOCK */ - static inline void vma_assert_can_modify(struct vm_area_struct *vma) { if (vma_is_attached(vma)) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 94f9c3ff5416..8e4e0bda3b58 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -96,25 +96,6 @@ #define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) -/* - * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to - * be naturally aligned with regard to the folio size. - * - * HVO which is only active if the size of struct page is a power of 2. - */ -#define MAX_FOLIO_VMEMMAP_ALIGN \ - (IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && \ - is_power_of_2(sizeof(struct page)) ? \ - MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) - -/* - * vmemmap optimization (like HVO) is only possible for page orders that fill - * two or more pages with struct pages. - */ -#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page))) -#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1) -#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0) - enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, @@ -492,11 +473,14 @@ enum lruvec_flags { * folio->flags, masked by LRU_REFS_MASK. */ #define MAX_NR_TIERS 4U +#define LRU_TIER_MIN 0U +#define LRU_TIER_MAX (MAX_NR_TIERS - 1) #ifndef __GENERATING_BOUNDS_H #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) +#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH) /* * For folios accessed multiple times through file descriptors, @@ -635,35 +619,32 @@ struct lru_gen_mm_walk { * For each node, memcgs are divided into two generations: the old and the * young. For each generation, memcgs are randomly sharded into multiple bins * to improve scalability. For each bin, the hlist_nulls is virtually divided - * into three segments: the head, the tail and the default. + * into two segments: the tail and the default. * * An onlining memcg is added to the tail of a random bin in the old generation. * The eviction starts at the head of a random bin in the old generation. The * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes * the old generation, is incremented when all its bins become empty. * - * There are four operations: - * 1. MEMCG_LRU_HEAD, which moves a memcg to the head of a random bin in its - * current generation (old or young) and updates its "seg" to "head"; - * 2. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its + * There are three operations: + * 1. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its * current generation (old or young) and updates its "seg" to "tail"; - * 3. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old + * 2. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old * generation, updates its "gen" to "old" and resets its "seg" to "default"; - * 4. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the + * 3. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the * young generation, updates its "gen" to "young" and resets its "seg" to * "default". * * The events that trigger the above operations are: - * 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD; - * 2. The first attempt to reclaim a memcg below low, which triggers + * 1. The first attempt to reclaim a memcg below low, which triggers * MEMCG_LRU_TAIL; - * 3. The first attempt to reclaim a memcg offlined or below reclaimable size + * 2. The first attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_TAIL; - * 4. The second attempt to reclaim a memcg offlined or below reclaimable size + * 3. The second attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_YOUNG; - * 5. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; - * 6. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; - * 7. Offlining a memcg, which triggers MEMCG_LRU_OLD. + * 4. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; + * 5. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; + * 6. Offlining a memcg, which triggers MEMCG_LRU_OLD. * * Notes: * 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing @@ -696,7 +677,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg); void lru_gen_online_memcg(struct mem_cgroup *memcg); void lru_gen_offline_memcg(struct mem_cgroup *memcg); void lru_gen_release_memcg(struct mem_cgroup *memcg); -void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid); void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid); bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid); void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); @@ -737,10 +717,6 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg) { } -static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) -{ -} - static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid) { } @@ -1157,8 +1133,8 @@ struct zone { /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP - struct page *vmemmap_tails[NR_VMEMMAP_TAILS]; +#ifdef CONFIG_VMEMMAP_OPTIMIZATION + struct page **vmemmap_tails; #endif } ____cacheline_internodealigned_in_smp; @@ -1662,7 +1638,7 @@ extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn extern void lruvec_init(struct lruvec *lruvec); -static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) +static inline struct pglist_data *lruvec_pgdat(const struct lruvec *lruvec) { #ifdef CONFIG_MEMCG return lruvec->pgdat; @@ -2021,19 +1997,23 @@ struct mem_section { unsigned long section_mem_map; struct mem_section_usage *usage; +#ifdef CONFIG_VMEMMAP_OPTIMIZATION + /* + * Normally, sections hold regular (order-0) pages. However, for + * sections with HVO enabled, this tracks the compound page order + * to enable deduplication of redundant vmemmap pages. + */ + unsigned int compound_page_order; +#endif #ifdef CONFIG_PAGE_EXTENSION /* * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use * section. (see page_ext.h about this.) */ struct page_ext *page_ext; - unsigned long pad; #endif - /* - * WARNING: mem_section must be a power-of-2 in size for the - * calculation and use of SECTION_ROOT_MASK to make sense. - */ -}; +/* Sacrifice minor padding space for efficient lookup. */ +} __aligned(2 * sizeof(unsigned long)); #ifdef CONFIG_SPARSEMEM_EXTREME #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section)) @@ -2043,7 +2023,6 @@ struct mem_section { #define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT) #define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT) -#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1) #ifdef CONFIG_SPARSEMEM_EXTREME extern struct mem_section **mem_section; @@ -2067,7 +2046,7 @@ static inline struct mem_section *__nr_to_section(unsigned long nr) if (!mem_section || !mem_section[root]) return NULL; #endif - return &mem_section[root][nr & SECTION_ROOT_MASK]; + return &mem_section[root][nr % SECTIONS_PER_ROOT]; } /* @@ -2089,9 +2068,6 @@ enum { SECTION_IS_EARLY_BIT, #ifdef CONFIG_ZONE_DEVICE SECTION_TAINT_ZONE_DEVICE_BIT, -#endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT - SECTION_IS_VMEMMAP_PREINIT_BIT, #endif SECTION_MAP_LAST_BIT, }; @@ -2103,9 +2079,6 @@ enum { #ifdef CONFIG_ZONE_DEVICE #define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT) #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -#define SECTION_IS_VMEMMAP_PREINIT BIT(SECTION_IS_VMEMMAP_PREINIT_BIT) -#endif #define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1)) #define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT @@ -2153,28 +2126,20 @@ static inline int online_device_section(const struct mem_section *section) return section && ((section->section_mem_map & flags) == flags); } + +static inline struct zone *device_zone(int nid) +{ + return &NODE_DATA(nid)->node_zones[ZONE_DEVICE]; +} #else static inline int online_device_section(const struct mem_section *section) { return 0; } -#endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -static inline int preinited_vmemmap_section(const struct mem_section *section) -{ - return (section && - (section->section_mem_map & SECTION_IS_VMEMMAP_PREINIT)); -} - -void sparse_vmemmap_init_nid_early(int nid); -#else -static inline int preinited_vmemmap_section(const struct mem_section *section) -{ - return 0; -} -static inline void sparse_vmemmap_init_nid_early(int nid) +static inline struct zone *device_zone(int nid) { + return NULL; } #endif @@ -2241,9 +2206,6 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long } #endif -void sparse_init_early_section(int nid, struct page *map, unsigned long pnum, - unsigned long flags); - #ifndef CONFIG_HAVE_ARCH_PFN_VALID /** * pfn_valid - check if there is a valid memory map entry for a PFN @@ -2379,7 +2341,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #else -#define sparse_vmemmap_init_nid_early(_nid) do {} while (0) #define pfn_in_present_section pfn_valid #endif /* CONFIG_SPARSEMEM */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7a863572adce..b0ddc652e76c 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -44,10 +44,6 @@ * Consequently, PG_reserved for a page mapped into user space can indicate * the zero page, the vDSO, MMIO pages or device memory. * - * The PG_private bitflag is set on pagecache pages if they contain filesystem - * specific data (which is normally at page->private). It can be used by - * private allocations for its own usage. - * * During initiation of disk I/O, PG_locked is set. This bit is set before I/O * and cleared when writeback _starts_ or when read _completes_. PG_writeback * is set before writeback starts and cleared when it finishes. @@ -105,7 +101,7 @@ enum pageflags { PG_owner_2, /* Owner use. If pagecache, fs may use */ PG_arch_1, PG_reserved, - PG_private, /* If pagecache, has fs-private data */ + PG_folio, /* Do not use: reserved for folio identification */ PG_private_2, /* If pagecache, has fs aux data */ PG_reclaim, /* To be reclaimed asap */ PG_swapbacked, /* Page is backed by RAM/swap */ @@ -208,14 +204,13 @@ enum pageflags { static __always_inline bool compound_info_has_mask(void) { /* - * Limit mask usage to HugeTLB vmemmap optimization (HVO) where it - * makes a difference. + * Limit mask usage to HVO where it makes a difference. * * The approach with mask would work in the wider set of conditions, * but it requires validating that struct pages are naturally aligned * for all orders up to the MAX_FOLIO_ORDER, which can be tricky. */ - if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP)) + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) return false; return is_power_of_2(sizeof(struct page)); @@ -576,9 +571,19 @@ FOLIO_FLAG(swapbacked, FOLIO_HEAD_PAGE) /* * Private page markings that may be used by the filesystem that owns the page * for its own purposes. - * - PG_private and PG_private_2 cause release_folio() and co to be invoked + * - folio->private and PG_private_2 cause release_folio() and co to be invoked */ -PAGEFLAG(Private, private, PF_ANY) + +static __always_inline bool folio_test_private(const struct folio *folio) +{ + /* + * data_race() is added for readers without holding the folio lock. + * Only the NULL/non-NULL answer is used and both are valid while + * private is being attached or detached, so the race is benign. + */ + return data_race(folio->private); +} + FOLIO_FLAG(private_2, FOLIO_HEAD_PAGE) /* owner_2 can be set on tail pages for anon memory */ @@ -588,13 +593,12 @@ FOLIO_FLAG(owner_2, FOLIO_HEAD_PAGE) * Only test-and-set exist for PG_writeback. The unconditional operators are * risky: they bypass page accounting. */ -TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL) - TESTSCFLAG(Writeback, writeback, PF_NO_TAIL) +FOLIO_TEST_FLAG(writeback, FOLIO_HEAD_PAGE) + FOLIO_TEST_SET_FLAG(writeback, FOLIO_HEAD_PAGE) FOLIO_FLAG(mappedtodisk, FOLIO_HEAD_PAGE) /* PG_readahead is only used for reads; PG_reclaim is only for writes */ -PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL) - TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL) +FOLIO_FLAG(reclaim, FOLIO_HEAD_PAGE) FOLIO_FLAG(readahead, FOLIO_HEAD_PAGE) FOLIO_TEST_CLEAR_FLAG(readahead, FOLIO_HEAD_PAGE) @@ -656,6 +660,7 @@ TESTSCFLAG(HWPoison, hwpoison, PF_ANY) #define __PG_HWPOISON (1UL << PG_hwpoison) #else PAGEFLAG_FALSE(HWPoison, hwpoison) +TESTSCFLAG_FALSE(HWPoison, hwpoison) #define __PG_HWPOISON 0 #endif @@ -1170,7 +1175,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) */ #define PAGE_FLAGS_CHECK_AT_FREE \ (1UL << PG_lru | 1UL << PG_locked | \ - 1UL << PG_private | 1UL << PG_private_2 | \ + 1UL << PG_private_2 | \ 1UL << PG_writeback | 1UL << PG_reserved | \ 1UL << PG_active | \ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) @@ -1194,8 +1199,31 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) (0xffUL /* order */ | 1UL << PG_has_hwpoisoned | \ 1UL << PG_large_rmappable | 1UL << PG_partially_mapped) -#define PAGE_FLAGS_PRIVATE \ - (1UL << PG_private | 1UL << PG_private_2) +/** + * folio_has_attached_private - check if the folio has private data attached + * @folio: The folio to check. + * + * Use this in code that may encounter swapcache or hugetlb folios but only + * wants to detect attached private data. + * + * Return: true if the folio has private data attached. + */ +static inline bool folio_has_attached_private(const struct folio *folio) +{ + /* + * Swapcache stores swp_entry_t in folio->swap, a union with + * folio->private, and hugetlb stores its own flags in folio->private; + * both are excluded. + * + * NOTE: For swapcache, folio->swap.val PG_swapcache are not set as + * a whole, so folio_test_swapcache() is not reliable to exclude + * swapcache. Use folio_test_swapbacked() instead, since it remains set + * when a folio is added to/removed from swapcache. + */ + + return folio_test_private(folio) && !folio_test_swapbacked(folio) && + !folio_test_hugetlb(folio); +} /** * folio_has_private - Determine if folio has private stuff * @folio: The folio to be checked @@ -1205,7 +1233,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) */ static inline int folio_has_private(const struct folio *folio) { - return !!(folio->flags.f & PAGE_FLAGS_PRIVATE); + return folio_has_attached_private(folio) || folio_test_private_2(folio); } #undef PF_ANY diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index d649b6bbbc87..2baf7a2b29b2 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -63,11 +63,12 @@ static inline void page_counter_init(struct page_counter *counter, counter->track_failcnt = false; } -static inline unsigned long page_counter_read(struct page_counter *counter) +static inline unsigned long page_counter_read(const struct page_counter *counter) { return atomic_long_read(&counter->usage); } +long page_counter_margin(const struct page_counter *counter); void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages); void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); bool page_counter_try_charge(struct page_counter *counter, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 0adfa6605653..bcbb0afe1a68 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -594,7 +594,6 @@ static inline void folio_attach_private(struct folio *folio, void *data) { folio_get(folio); folio->private = data; - folio_set_private(folio); } /** @@ -629,9 +628,8 @@ static inline void *folio_detach_private(struct folio *folio) { void *data = folio_get_private(folio); - if (!folio_test_private(folio)) + if (!data) return NULL; - folio_clear_private(folio); folio->private = NULL; folio_put(folio); @@ -1128,8 +1126,7 @@ static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, const pgoff_t pgoff = __linear_anon_page_index(vma, address); VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); - /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ - if (vma_is_anonymous(vma) && !vma->vm_file) + if (vma_is_anonymous(vma)) VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); return pgoff; @@ -1416,6 +1413,7 @@ struct readahead_control { bool dropbehind; bool _workingset; unsigned long _pflags; + bool _forward; }; #define DEFINE_READAHEAD(ractl, f, r, m, i) \ @@ -1480,18 +1478,29 @@ void page_cache_async_readahead(struct address_space *mapping, page_cache_async_ra(&ractl, folio, req_count); } +/* + * Adjust readahead_control to ensure next folio comes from + * [_index, _index + _nr_pages) afterwards and reset _batch_count. + */ +static inline void __readahead_advance(struct readahead_control *rac) +{ + if (rac->_forward) + rac->_index += rac->_batch_count; + + rac->_nr_pages -= rac->_batch_count; + rac->_batch_count = 0; +} + static inline struct folio *__readahead_folio(struct readahead_control *ractl) { struct folio *folio; BUG_ON(ractl->_batch_count > ractl->_nr_pages); - ractl->_nr_pages -= ractl->_batch_count; - ractl->_index += ractl->_batch_count; + __readahead_advance(ractl); + ractl->_forward = true; - if (!ractl->_nr_pages) { - ractl->_batch_count = 0; + if (!ractl->_nr_pages) return NULL; - } folio = xa_load(&ractl->mapping->i_pages, ractl->_index); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); @@ -1517,6 +1526,39 @@ static inline struct folio *readahead_folio(struct readahead_control *ractl) return folio; } +/** + * readahead_folio_last - Get the next folio to read, from the tail. + * @ractl: The current readahead request. + * + * Like readahead_folio(), but walks the range back-to-front. The folio is + * returned locked with its refcount dropped; the caller unlocks it once I/O + * completes. Compound folios are returned once, at their head index. + * + * Context: The folio is locked. + * Return: A pointer to the next folio, or %NULL when done. + */ +static inline struct folio *readahead_folio_last(struct readahead_control *ractl) +{ + struct folio *folio; + + /* Drop the previously returned batch from the remaining range. */ + __readahead_advance(ractl); + ractl->_forward = false; + + if (!ractl->_nr_pages) + return NULL; + + /* xa_load() follows sibling entries, so a tail index returns the head */ + folio = xa_load(&ractl->mapping->i_pages, + ractl->_index + ractl->_nr_pages - 1); + VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); + + ractl->_batch_count = folio_nr_pages(folio); + + folio_put(folio); + return folio; +} + static inline unsigned int __readahead_batch(struct readahead_control *rac, struct page **array, unsigned int array_sz) { @@ -1525,9 +1567,8 @@ static inline unsigned int __readahead_batch(struct readahead_control *rac, struct folio *folio; BUG_ON(rac->_batch_count > rac->_nr_pages); - rac->_nr_pages -= rac->_batch_count; - rac->_index += rac->_batch_count; - rac->_batch_count = 0; + __readahead_advance(rac); + rac->_forward = true; xas_set(&xas, rac->_index); rcu_read_lock(); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 8c093c119e5a..780fe849ff8b 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -490,35 +490,35 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma, #endif #ifndef ptep_get -static inline pte_t ptep_get(pte_t *ptep) +static inline pte_t ptep_get(const pte_t *ptep) { return READ_ONCE(*ptep); } #endif #ifndef pmdp_get -static inline pmd_t pmdp_get(pmd_t *pmdp) +static inline pmd_t pmdp_get(const pmd_t *pmdp) { return READ_ONCE(*pmdp); } #endif #ifndef pudp_get -static inline pud_t pudp_get(pud_t *pudp) +static inline pud_t pudp_get(const pud_t *pudp) { return READ_ONCE(*pudp); } #endif #ifndef p4dp_get -static inline p4d_t p4dp_get(p4d_t *p4dp) +static inline p4d_t p4dp_get(const p4d_t *p4dp) { return READ_ONCE(*p4dp); } #endif #ifndef pgdp_get -static inline pgd_t pgdp_get(pgd_t *pgdp) +static inline pgd_t pgdp_get(const pgd_t *pgdp) { return READ_ONCE(*pgdp); } @@ -2313,6 +2313,20 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } +void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size); + +#define ptval_to_str(buf, val) \ + do { \ + auto __val = (val); \ + \ + ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ + } while (0) + +#if defined(__SIZEOF_INT128__) +#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ +#else +#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ +#endif #endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) diff --git a/include/linux/sched.h b/include/linux/sched.h index 12a7e094a3e6..1c93da75226e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1814,8 +1814,8 @@ extern struct pid __rcu *cad_pid; #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ #define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */ #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ -#define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ -#define PF_KSWAPD 0x00020000 /* I am kswapd */ +#define PF__HOLE__00010000 0x00010000 +#define PF__HOLE__00020000 0x00020000 #define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */ #define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */ #define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to, diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3030d9245f5a..27c32fbabfec 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -5,16 +5,92 @@ #ifndef _LINUX_SET_MEMORY_H_ #define _LINUX_SET_MEMORY_H_ +/** + * DOC: Kernel page table permissions + * + * The set_memory() and set_direct_map() APIs update permissions of existing + * kernel mappings. + * + * The set_memory() functions operate on a range of kernel virtual addresses, + * the set_direct_map() functions operate on the direct map. + * + * The updates are not atomic: when a call fails, an arbitrary prefix of the + * range may have been updated already and there is no automatic rollback. + * A caller must restore the required permissions before reusing or freeing + * the memory. + * + * When an architecture does not implement these APIs they succeed without + * doing anything, so a return value of 0 does not mean that the permissions + * were actually changed. + * + * Callers that depend on the permissions being applied must ensure that the + * architecture supports the required operation for the target addresses. The + * Kconfig symbols alone do not guarantee this. + * + * See Documentation/mm/kernel-page-tables.rst for the details and for the + * differences between the architecture implementations. + */ + #ifdef CONFIG_ARCH_HAS_SET_MEMORY #include #else +/** + * set_memory_ro - make a kernel mapping read-only + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_ro(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_rw - make a kernel mapping writable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_rw(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_x - make a kernel mapping executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_x(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_nx - make a kernel mapping non-executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_nx(unsigned long addr, int numpages) { return 0; } #endif #ifndef set_memory_rox +/** + * set_memory_rox - make a kernel mapping read-only and executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * A failure may leave the range read-only but not executable. + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int set_memory_rox(unsigned long addr, int numpages) { int ret = set_memory_ro(addr, numpages); @@ -25,17 +101,35 @@ static inline int set_memory_rox(unsigned long addr, int numpages) #endif #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP -static inline int set_direct_map_invalid_noflush(struct page *page) -{ - return 0; -} -static inline int set_direct_map_default_noflush(struct page *page) +/** + * set_direct_map_invalid_noflush - remove pages from the direct map + * @page: first page to update + * @nr: number of pages to update + * + * Makes the direct mapping of @nr pages starting at @page not present. + * The caller is responsible for any required TLB flushing. + * + * Return: 0 on success, negative error code on failure. + */ +static inline int set_direct_map_invalid_noflush(struct page *page, + unsigned int nr) { return 0; } -static inline int set_direct_map_valid_noflush(struct page *page, - unsigned nr, bool valid) +/** + * set_direct_map_default_noflush - restore the direct map of pages + * @page: first page to update + * @nr: number of pages to update + * + * Restores the default kernel permissions of the direct mapping of @nr + * pages starting at @page. + * The caller is responsible for any required TLB flushing. + * + * Return: 0 on success, negative error code on failure. + */ +static inline int set_direct_map_default_noflush(struct page *page, + unsigned int nr) { return 0; } @@ -50,6 +144,18 @@ static inline bool kernel_page_present(struct page *page) * boot time. Let them overrive this query. */ #ifndef can_set_direct_map +/** + * can_set_direct_map - check if the direct map can be modified + * + * Available with CONFIG_ARCH_HAS_SET_DIRECT_MAP. Architectures may override + * this to report whether direct map updates are enabled at runtime. + * Even though the generic implementation returns true this does not guarantee + * that every address can be updated. + * + * See Documentation/mm/kernel-page-tables.rst for the details + * + * Return: true unless the architecture reports direct map updates disabled. + */ static inline bool can_set_direct_map(void) { return true; diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 5663dff53186..a7c7a96a7cbf 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -11,6 +11,7 @@ #include #include #include +#include /* inode in-kernel data */ @@ -54,6 +55,11 @@ struct shmem_inode_info { struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif struct inode vfs_inode; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct obj_cgroup *shrinklist_objcg; + int shrinklist_nid; +#endif }; #define SHMEM_FL_USER_VISIBLE (FS_FL_USER_VISIBLE | FS_CASEFOLD_FL) @@ -83,9 +89,9 @@ struct shmem_sb_info { ino_t next_ino; /* The next per-sb inode number to use */ ino_t __percpu *ino_batch; /* The next per-cpu inode number to use */ struct mempolicy *mpol; /* default memory policy for mappings */ - spinlock_t shrinklist_lock; /* Protects shrinklist */ - struct list_head shrinklist; /* List of shinkable inodes */ - unsigned long shrinklist_len; /* Length of shrinklist */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct list_lru shrinklist; /* List of shrinkable inodes */ +#endif struct shmem_quota_limits qlimits; /* Default quota limits */ struct simple_xattr_cache xa_cache; }; diff --git a/include/linux/string.h b/include/linux/string.h index 5702daca4326..6cb5cdd01158 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -278,6 +278,19 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) } #endif +#ifndef memcpy_nontemporal +/* + * memcpy_nontemporal() requests a non-temporal copy when the + * architecture has a suitable backend. Architectures without a + * specialized backend fall back to memcpy(). Keep this as a + * function-like macro so the compiler can still see the original + * memcpy() call site and preserve the usual FORTIFY coverage when + * object sizes remain visible there, while keeping the API void. + */ +#define memcpy_nontemporal(dst, src, len) \ + ((void)memcpy(dst, src, len)) +#endif + void *memchr_inv(const void *s, int c, size_t n); char *strreplace(char *str, char old, char new); diff --git a/include/linux/swap.h b/include/linux/swap.h index 5658a1634b85..cb434cccd653 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -25,12 +25,6 @@ #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) - -static inline int current_is_kswapd(void) -{ - return current->flags & PF_KSWAPD; -} - /* * MAX_SWAPFILES defines the maximum number of swaptypes: things which can * be swapped to. The swap type and the offset into that swap type are @@ -246,7 +240,7 @@ struct swap_info_struct { struct plist_node list; /* entry in swap_active_head */ signed char type; /* strange name for an index */ unsigned int max; /* size of this swap device */ - struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */ + struct swap_cluster_info *cluster_info; /* array, one entry per cluster */ struct list_head free_clusters; /* free clusters list */ struct list_head full_clusters; /* full clusters list */ struct list_head nonfull_clusters[SWAP_NR_ORDERS]; @@ -278,15 +272,39 @@ struct swap_info_struct { const struct swap_ops *ops; }; -static inline swp_entry_t page_swap_entry(struct page *page) +/** + * folio_swap_entry - Return the swap entry at a page index within a folio. + * @folio: The folio. + * @idx: The index of the page within the folio. + * + * A folio in the swap cache occupies folio_nr_pages() contiguous swap + * entries starting at folio->swap. The caller must ensure the folio is + * in the swap cache and that @idx is within the folio. + */ +static inline +swp_entry_t folio_swap_entry(const struct folio *folio, unsigned long idx) { - struct folio *folio = page_folio(page); swp_entry_t entry = folio->swap; - entry.val += folio_page_idx(folio, page); + VM_WARN_ON_ONCE_FOLIO(idx >= folio_nr_pages(folio), folio); + entry.val += idx; return entry; } +/** + * folio_page_swap_entry - Return the swap entry of a page in a folio. + * @folio: The folio containing @page. + * @page: A page within @folio. + * + * The caller must ensure the folio is in the swap cache and that @page + * is part of @folio. + */ +static inline swp_entry_t folio_page_swap_entry(const struct folio *folio, + const struct page *page) +{ + return folio_swap_entry(folio, folio_page_idx(folio, page)); +} + /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; @@ -318,6 +336,9 @@ static inline bool lru_cache_disabled(void) extern unsigned long shrink_all_memory(unsigned long nr_pages); long remove_mapping(struct address_space *mapping, struct folio *folio); +long remove_mapping_set_shadow(struct address_space *mapping, + struct folio *folio, + struct mem_cgroup *target_memcg); #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) extern int reclaim_register_node(struct node *node); @@ -339,6 +360,7 @@ void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); extern void __meminit kswapd_stop(int nid); +bool current_is_kswapd(void); #ifdef CONFIG_SWAP int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, @@ -402,6 +424,8 @@ void swap_put_entries_direct(swp_entry_t entry, int nr); */ bool folio_free_swap(struct folio *folio); +void swap_writeback_dropbehind_folio(struct folio *folio); + /* Allocate / free (hibernation) exclusive entries */ swp_entry_t swap_alloc_hibernation_slot(int type); void swap_free_hibernation_slot(swp_entry_t entry); @@ -412,6 +436,7 @@ static inline void put_swap_device(struct swap_info_struct *si) } #else /* CONFIG_SWAP */ +static inline void swap_writeback_dropbehind_folio(struct folio *folio) {} static inline struct swap_info_struct *get_swap_device(swp_entry_t entry) { return NULL; @@ -508,8 +533,9 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_p __mem_cgroup_uncharge_swap(id, nr_pages); } -extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg); -extern bool mem_cgroup_swap_full(struct folio *folio); +long mem_cgroup_get_folio_swap_margin(const struct folio *folio); +long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg); +bool mem_cgroup_swap_full(const struct folio *folio); #else static inline int mem_cgroup_try_charge_swap(struct folio *folio) { @@ -521,12 +547,17 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, { } -static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) +static inline long mem_cgroup_get_folio_swap_margin(const struct folio *folio) +{ + return PAGE_COUNTER_MAX; +} + +static inline long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg) { return get_nr_swap_pages(); } -static inline bool mem_cgroup_swap_full(struct folio *folio) +static inline bool mem_cgroup_swap_full(const struct folio *folio) { return vm_swap_full(); } diff --git a/include/linux/swapops.h b/include/linux/swapops.h index e7d0d529f3e0..603f9e3f909c 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -261,11 +261,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page) return swp_entry(SWP_HWPOISON, page_to_pfn(page)); } -static inline int is_hwpoison_entry(swp_entry_t entry) -{ - return swp_type(entry) == SWP_HWPOISON; -} - #else static inline swp_entry_t make_hwpoison_entry(struct page *page) @@ -273,10 +268,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page) return swp_entry(0, 0); } -static inline int is_hwpoison_entry(swp_entry_t swp) -{ - return 0; -} #endif typedef unsigned long pte_marker; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index aed121d729b0..034a693777ca 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -3,6 +3,7 @@ #define _LINUX_VMALLOC_H #include +#include #include #include #include @@ -38,6 +39,7 @@ struct iov_iter; /* in uio.h */ #define VM_DEFER_KMEMLEAK 0 #endif #define VM_SPARSE 0x00001000 /* sparse vm_area. not all pages are present. */ +#define VM_REQUIRE_HUGE_VMAP 0x00002000 /* huge page mapping or nothing */ /* bits [20..32] reserved for arch specific ioremap internals */ @@ -214,6 +216,8 @@ void *__must_check vrealloc_node_align_noprof(const void *p, size_t size, extern void vfree(const void *addr); extern void vfree_atomic(const void *addr); +DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T)) + extern void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot); void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot); diff --git a/include/linux/vmemmap-optimization.h b/include/linux/vmemmap-optimization.h new file mode 100644 index 000000000000..fa9e9abd6656 --- /dev/null +++ b/include/linux/vmemmap-optimization.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * vmemmap-optimization.h + * + * Generic vmemmap optimization declarations. + * + * Author: Muchun Song + */ +#ifndef _LINUX_VMEMMAP_OPTIMIZATION_H +#define _LINUX_VMEMMAP_OPTIMIZATION_H + +#include +#include +#include +#include + +/* + * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to + * be naturally aligned with regard to the folio size. + * + * HVO which is only active if the size of struct page is a power of 2. + */ +#define MAX_FOLIO_VMEMMAP_ALIGN \ + (IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \ + is_power_of_2(sizeof(struct page)) ? \ + MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) + +/* The number of retained vmemmap pages with HVO enabled. */ +#define VMEMMAP_OPTIMIZATION_PAGES 1 +#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \ + (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) +#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) + +#ifdef CONFIG_VMEMMAP_OPTIMIZATION +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return section->compound_page_order; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ + VM_WARN_ON(section_compound_order(section) && order && + section_compound_order(section) != order); + section->compound_page_order = order; +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ + unsigned long section_nr = pfn_to_section_nr(pfn); + + if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)) + return; + + for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++) + section_set_compound_order(__nr_to_section(section_nr + i), order); +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return section_compound_order(__pfn_to_section(pfn)); +} + +struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); +#else +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return 0; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return 0; +} + +static inline struct page *vmemmap_shared_tail_page(unsigned int order, + struct zone *zone) +{ + return NULL; +} +#endif /* CONFIG_VMEMMAP_OPTIMIZATION */ + +static inline bool vmemmap_optimizable_pfn(unsigned long pfn) +{ + const unsigned int order = pfn_to_section_compound_order(pfn); + const unsigned long nr_pages = 1UL << order; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; +} + +static inline bool vmemmap_optimizable_order(unsigned int order) +{ + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) + return false; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER; +} +#endif /* _LINUX_VMEMMAP_OPTIMIZATION_H */ diff --git a/include/linux/zsmalloc.h b/include/linux/zsmalloc.h index 478410c880b1..5b7298a5026b 100644 --- a/include/linux/zsmalloc.h +++ b/include/linux/zsmalloc.h @@ -40,10 +40,6 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size); void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats); -void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *local_copy); -void zs_obj_read_end(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *handle_mem); void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, struct scatterlist *sg, size_t mem_len); void zs_obj_read_sg_end(struct zs_pool *pool, unsigned long handle); diff --git a/include/linux/zswap.h b/include/linux/zswap.h index 30c193a1207e..df6cafbe95dc 100644 --- a/include/linux/zswap.h +++ b/include/linux/zswap.h @@ -27,7 +27,7 @@ struct zswap_lruvec_state { unsigned long zswap_total_pages(void); bool zswap_store(struct folio *folio); int zswap_load(struct folio *folio); -void zswap_invalidate(swp_entry_t swp); +void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries); int zswap_swapon(int type, unsigned long nr_pages); void zswap_swapoff(int type); void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg); @@ -49,7 +49,11 @@ static inline int zswap_load(struct folio *folio) return -ENOENT; } -static inline void zswap_invalidate(swp_entry_t swp) {} +static inline void zswap_invalidate(int type, pgoff_t offset, + unsigned long nr_entries) +{ +} + static inline int zswap_swapon(int type, unsigned long nr_pages) { return 0; diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h index e1fa9131c490..71d726b1f784 100644 --- a/include/net/mana/mana.h +++ b/include/net/mana/mana.h @@ -47,8 +47,8 @@ enum mana_priv_flag_bits { #define COMP_ENTRY_SIZE 64 /* This Max value for RX buffers is derived from __alloc_page()'s max page - * allocation calculation. It allows maximum 2^(MAX_ORDER -1) pages. RX buffer - * size beyond this value gets rejected by __alloc_page() call. + * allocation calculation. It allows maximum 2^MAX_PAGE_ORDER pages. RX + * buffer size beyond this value gets rejected by __alloc_page() call. */ #define MAX_RX_BUFFERS_PER_QUEUE 8192 #define DEF_RX_BUFFERS_PER_QUEUE 1024 diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 5a48c5406cce..5fb4d92cfd84 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -55,10 +55,10 @@ SCAN_STATUS TRACE_EVENT(mm_khugepaged_scan_pmd, - TP_PROTO(struct mm_struct *mm, struct folio *folio, + TP_PROTO(struct mm_struct *mm, unsigned long pfn, int referenced, int none_or_zero, int status, int unmapped), - TP_ARGS(mm, folio, referenced, none_or_zero, status, unmapped), + TP_ARGS(mm, pfn, referenced, none_or_zero, status, unmapped), TP_STRUCT__entry( __field(struct mm_struct *, mm) @@ -71,7 +71,7 @@ TRACE_EVENT(mm_khugepaged_scan_pmd, TP_fast_assign( __entry->mm = mm; - __entry->pfn = folio ? folio_pfn(folio) : -1; + __entry->pfn = pfn; __entry->referenced = referenced; __entry->none_or_zero = none_or_zero; __entry->status = status; @@ -178,10 +178,10 @@ TRACE_EVENT(mm_collapse_huge_page_swapin, TRACE_EVENT(mm_khugepaged_scan_file, - TP_PROTO(struct mm_struct *mm, struct folio *folio, struct file *file, + TP_PROTO(struct mm_struct *mm, unsigned long pfn, struct file *file, int present, int swap, int result), - TP_ARGS(mm, folio, file, present, swap, result), + TP_ARGS(mm, pfn, file, present, swap, result), TP_STRUCT__entry( __field(struct mm_struct *, mm) @@ -194,7 +194,7 @@ TRACE_EVENT(mm_khugepaged_scan_file, TP_fast_assign( __entry->mm = mm; - __entry->pfn = folio ? folio_pfn(folio) : -1; + __entry->pfn = pfn; __assign_str(filename); __entry->present = present; __entry->swap = swap; @@ -211,10 +211,10 @@ TRACE_EVENT(mm_khugepaged_scan_file, ); TRACE_EVENT(mm_khugepaged_collapse_file, - TP_PROTO(struct mm_struct *mm, struct folio *new_folio, pgoff_t index, + TP_PROTO(struct mm_struct *mm, unsigned long new_pfn, pgoff_t index, unsigned long addr, bool is_shmem, struct file *file, int nr, int result), - TP_ARGS(mm, new_folio, index, addr, is_shmem, file, nr, result), + TP_ARGS(mm, new_pfn, index, addr, is_shmem, file, nr, result), TP_STRUCT__entry( __field(struct mm_struct *, mm) __field(unsigned long, hpfn) @@ -228,7 +228,7 @@ TRACE_EVENT(mm_khugepaged_collapse_file, TP_fast_assign( __entry->mm = mm; - __entry->hpfn = new_folio ? folio_pfn(new_folio) : -1; + __entry->hpfn = new_pfn; __entry->index = index; __entry->addr = addr; __entry->is_shmem = is_shmem; diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 935893e5ea53..3c153b3ad845 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -144,7 +144,7 @@ TRACE_DEFINE_ENUM(___GFP_LAST_BIT); DEF_PAGEFLAG_NAME(owner_2), \ DEF_PAGEFLAG_NAME(arch_1), \ DEF_PAGEFLAG_NAME(reserved), \ - DEF_PAGEFLAG_NAME(private), \ + DEF_PAGEFLAG_NAME(folio), \ DEF_PAGEFLAG_NAME(private_2), \ DEF_PAGEFLAG_NAME(writeback), \ DEF_PAGEFLAG_NAME(head), \ @@ -168,6 +168,10 @@ IF_HAVE_PG_ARCH_3(arch_3) #define __VM_ARCH_SPECIFIC_1 {VM_SAO, "sao" } #elif defined(CONFIG_PARISC) #define __VM_ARCH_SPECIFIC_1 {VM_GROWSUP, "growsup" } +#elif defined(CONFIG_SPARC64) +#define __VM_ARCH_SPECIFIC_1 {VM_SPARC_ADI, "adi" } +#elif defined(CONFIG_ARM64) +#define __VM_ARCH_SPECIFIC_1 {VM_ARM64_BTI, "bti" } #elif !defined(CONFIG_MMU) #define __VM_ARCH_SPECIFIC_1 {VM_MAPPED_COPY,"mappedcopy" } #else @@ -198,6 +202,37 @@ IF_HAVE_PG_ARCH_3(arch_3) # define IF_HAVE_VM_DROPPABLE(flag, name) #endif +#ifdef CONFIG_ARCH_HAS_PKEYS +# define IF_HAVE_VM_PKEY(flag, name) {flag, name}, +#if CONFIG_ARCH_PKEY_BITS > 3 +# define IF_HAVE_VM_PKEY3(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_PKEY3(flag, name) +#endif +#if CONFIG_ARCH_PKEY_BITS > 4 +# define IF_HAVE_VM_PKEY4(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_PKEY4(flag, name) +#endif +#else +# define IF_HAVE_VM_PKEY(flag, name) +# define IF_HAVE_VM_PKEY3(flag, name) +# define IF_HAVE_VM_PKEY4(flag, name) +#endif + +#ifdef CONFIG_ARM64_MTE +# define IF_HAVE_VM_MTE(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_MTE(flag, name) +#endif + +#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI) || \ + defined(CONFIG_ARM64_GCS) +# define IF_HAVE_VM_SHADOW_STACK(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_SHADOW_STACK(flag, name) +#endif + #define __def_vmaflag_names \ {VM_READ, "read" }, \ {VM_WRITE, "write" }, \ @@ -233,6 +268,14 @@ IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY, "softdirty" ) \ {VM_HUGEPAGE, "hugepage" }, \ {VM_NOHUGEPAGE, "nohugepage" }, \ IF_HAVE_VM_DROPPABLE(VM_DROPPABLE, "droppable" ) \ +IF_HAVE_VM_PKEY(VM_PKEY_BIT0, "pkey_bit0") \ +IF_HAVE_VM_PKEY(VM_PKEY_BIT1, "pkey_bit1") \ +IF_HAVE_VM_PKEY(VM_PKEY_BIT2, "pkey_bit2") \ +IF_HAVE_VM_PKEY3(VM_PKEY_BIT3, "pkey_bit3") \ +IF_HAVE_VM_PKEY4(VM_PKEY_BIT4, "pkey_bit4") \ +IF_HAVE_VM_MTE(VM_MTE, "mte") \ +IF_HAVE_VM_MTE(VM_MTE_ALLOWED, "mte_allowed") \ +IF_HAVE_VM_SHADOW_STACK(VM_SHADOW_STACK, "shadow_stack") \ {VM_MERGEABLE, "mergeable" } \ #define show_vma_flags(flags) \ diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 36c3a90f0acc..5d47b774633a 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h @@ -22,7 +22,7 @@ (folio_test_swapcache(folio) ? PAGEMAP_SWAPCACHE : 0) | \ (folio_test_swapbacked(folio) ? PAGEMAP_SWAPBACKED : 0) | \ (folio_test_mappedtodisk(folio) ? PAGEMAP_MAPPEDDISK : 0) | \ - (folio_test_private(folio) ? PAGEMAP_BUFFERS : 0) \ + (folio_has_attached_private(folio) ? PAGEMAP_BUFFERS : 0) \ ) TRACE_EVENT(mm_lru_insertion, diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h index b4bf7b8def1f..8a872990b4be 100644 --- a/include/trace/events/vmscan.h +++ b/include/trace/events/vmscan.h @@ -214,13 +214,6 @@ DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_reclaim_be TP_ARGS(gfp_flags, order, memcg) ); - -DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_softlimit_reclaim_begin, - - TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg), - - TP_ARGS(gfp_flags, order, memcg) -); #endif /* CONFIG_MEMCG */ DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_end_template, @@ -260,13 +253,6 @@ DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_reclaim_end, TP_ARGS(nr_reclaimed, memcg) ); - -DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_softlimit_reclaim_end, - - TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg), - - TP_ARGS(nr_reclaimed, memcg) -); #endif /* CONFIG_MEMCG */ TRACE_EVENT(mm_shrink_slab_start, diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index d09d4c3fe547..f7e8d766d128 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -272,13 +272,10 @@ struct stack_map_vma_lock { /* * Acquire a stable read-side reference on the VMA covering @ip. * - * With CONFIG_PER_VMA_LOCK=y this returns a VMA with its per-VMA read - * lock held and mmap_lock dropped, so the caller may sleep. - * - * With CONFIG_PER_VMA_LOCK=n it returns a VMA with mmap_lock still - * held; the caller must snapshot any fields it needs and pin vm_file - * with get_file() before stack_map_unlock_vma() drops mmap_lock, as - * the VMA may be split, merged, or freed after that. + * On NOMMU configurations, returns with the mmap_lock held. If the MMU + * is enabled, the per-VMA lock will be held instead. The lock + * should be released with stack_map_unlock_vma() which will release the + * appropriate lock. Once the lock is released, the VMA may be freed. * * Returns NULL on failure, in which case no lock is held. */ @@ -288,7 +285,6 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) struct mm_struct *mm = lock->mm; struct vm_area_struct *vma; - /* noop under !CONFIG_PER_VMA_LOCK */ vma = lock_vma_under_rcu(mm, ip); if (vma) { lock->vma = vma; @@ -308,21 +304,20 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) return NULL; } -#ifdef CONFIG_PER_VMA_LOCK +#ifdef CONFIG_MMU if (!vma_start_read_locked(vma)) { mmap_read_unlock(mm); return NULL; } mmap_read_unlock(mm); #endif - lock->vma = vma; return vma; } static void stack_map_unlock_vma(struct stack_map_vma_lock *lock) { -#ifdef CONFIG_PER_VMA_LOCK +#ifdef CONFIG_MMU vma_end_read(lock->vma); #else mmap_read_unlock(lock->mm); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 13e1aabe6f88..c65ba1dcd866 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -869,7 +869,7 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it, BUILD_BUG_ON(sizeof(struct bpf_iter_task_vma_kern) != sizeof(struct bpf_iter_task_vma)); BUILD_BUG_ON(__alignof__(struct bpf_iter_task_vma_kern) != __alignof__(struct bpf_iter_task_vma)); - if (!IS_ENABLED(CONFIG_PER_VMA_LOCK)) { + if (!IS_ENABLED(CONFIG_MMU)) { kit->data = NULL; return -EOPNOTSUPP; } diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index f2684f9f8e0a..233fb8b45d2c 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -285,7 +285,6 @@ extern struct kernfs_syscall_ops cgroup1_kf_syscall_ops; extern const struct fs_parameter_spec cgroup1_fs_parameters[]; int proc_cgroupstats_show(struct seq_file *m, void *v); -bool cgroup1_ssid_disabled(int ssid); void cgroup1_pidlist_destroy_all(struct cgroup *cgrp); void cgroup1_release_agent(struct work_struct *work); void cgroup1_check_for_release(struct cgroup *cgrp); diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 13bd42e5493d..64d4dbdc6c07 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -634,11 +634,10 @@ static struct page *rb_alloc_aux_page(int node, int order) /* * Communicate the allocation size to the driver: * if we managed to secure a high-order allocation, - * set its first page's private to this order; - * !PagePrivate(page) means it's just a normal page. + * set its first page's private to this order, otherwise page's + * private remains zero. */ split_page(page, order); - SetPagePrivate(page); set_page_private(page, order); } @@ -649,7 +648,7 @@ static void rb_free_aux_page(struct perf_buffer *rb, int idx) { struct page *page = virt_to_page(rb->aux_pages[idx]); - ClearPagePrivate(page); + set_page_private(page, 0); __free_page(page); } diff --git a/kernel/fork.c b/kernel/fork.c index 394906afafd3..2142c6f38119 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1083,9 +1083,7 @@ static void mmap_init_lock(struct mm_struct *mm) { init_rwsem(&mm->mmap_lock); mm_lock_seqcount_init(mm); -#ifdef CONFIG_PER_VMA_LOCK rcuwait_init(&mm->vma_writer_wait); -#endif } static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index b209712cb2c3..d5dba0e50b2e 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -88,7 +88,7 @@ static inline int hibernate_restore_unprotect_page(void *page_address) {return 0 static inline void hibernate_map_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { - int ret = set_direct_map_default_noflush(page); + int ret = set_direct_map_default_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); @@ -101,7 +101,7 @@ static inline void hibernate_unmap_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { unsigned long addr = (unsigned long)page_address(page); - int ret = set_direct_map_invalid_noflush(page); + int ret = set_direct_map_invalid_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c index 8614430ca212..5a417f8a922a 100644 --- a/kernel/vmcore_info.c +++ b/kernel/vmcore_info.c @@ -216,7 +216,6 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); VMCOREINFO_NUMBER(NR_FREE_PAGES); VMCOREINFO_NUMBER(PG_lru); - VMCOREINFO_NUMBER(PG_private); VMCOREINFO_NUMBER(PG_swapcache); VMCOREINFO_NUMBER(PG_swapbacked); #define PAGE_SLAB_MAPCOUNT_VALUE (PGTY_slab << 24) diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index b0b07270ce7c..06f77fb3179f 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -244,7 +244,7 @@ static int span_iteration_check(void) unsigned long start, last; struct interval_tree_span_iter span, mas_span; - DEFINE_MTREE(tree); + struct maple_tree tree; MA_STATE(mas, &tree, 0, 0); diff --git a/lib/region_alloc_benchmark.c b/lib/region_alloc_benchmark.c index e88b4cf55c62..a644f3d5431a 100644 --- a/lib/region_alloc_benchmark.c +++ b/lib/region_alloc_benchmark.c @@ -78,11 +78,13 @@ static size_t __init ida_size(unsigned long nr_ids) static unsigned long __init benchmark_ida(unsigned long cap) { - struct ida ida = IDA_INIT(ida); + struct ida ida; unsigned long cnt, idx, off, nr_ids = 0; ktime_t alloc_time, free_time; int id = -ENOSPC; + ida_init(&ida); + alloc_time = ktime_get(); for (cnt = 0; cnt <= cap; cnt++) { for (off = 0; off < reg_sz[cnt]; off++) { @@ -125,12 +127,14 @@ static unsigned long __init benchmark_ida(unsigned long cap) static unsigned long __init benchmark_maple_tree(unsigned long cap) { - struct maple_tree mt = MTREE_INIT(mt, MT_FLAGS_ALLOC_RANGE); + struct maple_tree mt; unsigned long cnt, idx; ktime_t alloc_time, free_time; size_t sz; int ret; + mt_init_flags(&mt, MT_FLAGS_ALLOC_RANGE); + alloc_time = ktime_get(); for (cnt = 0; cnt <= cap; cnt++) { ret = mtree_alloc_range(&mt, &idx, xa_mk_value(cnt + 1), diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 6911daa9f854..b409c42bfe6f 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -1624,6 +1624,8 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, if (cmd.addr & ~PAGE_MASK) return -EINVAL; + if (cmd.npages > ULONG_MAX >> PAGE_SHIFT) + return -EINVAL; if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) return -EINVAL; diff --git a/mm/Kconfig b/mm/Kconfig index 604c58199acb..edb4a6c0a870 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -461,8 +461,10 @@ config SPARSEMEM_VMEMMAP pfn_to_page and page_to_pfn operations. This is the most efficient option when sufficient kernel resources are available. -config SPARSEMEM_VMEMMAP_PREINIT +config VMEMMAP_OPTIMIZATION bool + depends on SPARSEMEM_VMEMMAP + # # Select this config option from the architecture Kconfig, if it is preferred # to enable the feature of HugeTLB/dev_dax vmemmap optimization. @@ -473,9 +475,6 @@ config ARCH_WANT_OPTIMIZE_DAX_VMEMMAP config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP bool -config ARCH_WANT_HUGETLB_VMEMMAP_PREINIT - bool - config HAVE_MEMBLOCK_PHYS_MAP bool @@ -1225,6 +1224,7 @@ config ZONE_DMA32 config ZONE_DEVICE bool "Device memory (pmem, HMM, etc...) hotplug support" depends on MEMORY_HOTREMOVE + select VMEMMAP_OPTIMIZATION if ARCH_WANT_OPTIMIZE_DAX_VMEMMAP select XARRAY_MULTI help @@ -1292,24 +1292,19 @@ config PERCPU_STATS be used to help understand percpu memory usage. config GUP_TEST - bool "Enable infrastructure for get_user_pages()-related unit tests" + bool "Enable infrastructure for get_user_pages()-related unit tests and benchmarks" depends on DEBUG_FS help Provides /sys/kernel/debug/gup_test, which in turn provides a way - to make ioctl calls that can launch kernel-based unit tests for - the get_user_pages*() and pin_user_pages*() family of API calls. + to make ioctl calls that can launch kernel-based unit tests and + benchmarks for the get_user_pages*() and pin_user_pages*() families + of API calls. - These tests include benchmark testing of the _fast variants of - get_user_pages*() and pin_user_pages*(), as well as smoke tests of + These include benchmark testing of the _fast variants of + get_user_pages*() and pin_user_pages*(), as well as tests of the non-_fast variants. - There is also a sub-test that allows running dump_page() on any - of up to eight pages (selected by command line args) within the - range of user-space addresses. These pages are either pinned via - pin_user_pages*(), or pinned via get_user_pages*(), as specified - by other command line arguments. - - See tools/testing/selftests/mm/gup_test.c + See tools/testing/selftests/mm/gup.c and tools/mm/gup_bench.c. comment "GUP_TEST needs to have DEBUG_FS enabled" depends on !GUP_TEST && !DEBUG_FS @@ -1430,18 +1425,6 @@ config LRU_GEN_WALKS_MMU depends on LRU_GEN && ARCH_HAS_HW_PTE_YOUNG # } -config ARCH_SUPPORTS_PER_VMA_LOCK - def_bool n - -config PER_VMA_LOCK - def_bool y - depends on ARCH_SUPPORTS_PER_VMA_LOCK && MMU && SMP - help - Allow per-vma locking during page fault handling. - - This feature allows locking each virtual memory area separately when - handling page faults instead of taking mmap_lock. - config LOCK_MM_AND_FIND_VMA bool depends on !STACK_GROWSUP @@ -1482,7 +1465,7 @@ config HAVE_ARCH_TLB_REMOVE_TABLE config PT_RECLAIM def_bool y - depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE + depends on MMU && !HAVE_ARCH_TLB_REMOVE_TABLE help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 15dca19dd07d..9eaa25d1cf23 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -310,7 +310,6 @@ config DEBUG_KMEMLEAK_VERBOSE config PER_VMA_LOCK_STATS bool "Statistics for per-vma locks" - depends on PER_VMA_LOCK help Say Y here to enable success, retry and failure counters of page faults handled under protection of per-vma locks. When enabled, the diff --git a/mm/Makefile b/mm/Makefile index e7245cb88c66..2a3ec53d62ee 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -55,7 +55,8 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ mm_init.o percpu.o slab_common.o \ compaction.o show_mem.o \ interval_tree.o list_lru.o workingset.o \ - debug.o gup.o mmap_lock.o vma_init.o $(mmu-y) + debug.o gup.o mmap_lock.o vma_init.o char-mem.o \ + $(mmu-y) # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c index b33410310477..82e2c3448dcf 100644 --- a/mm/alloc_tag.c +++ b/mm/alloc_tag.c @@ -621,7 +621,7 @@ void __init alloc_tag_sec_init(void) kernel_tags.count = last_codetag - kernel_tags.first_tag; /* Check if kernel tags fit into page flags */ - if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { + if (CODETAG_ID_FIRST + kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); diff --git a/drivers/char/mem.c b/mm/char-mem.c similarity index 97% rename from drivers/char/mem.c rename to mm/char-mem.c index c4c78182b56e..c5761ddedbd7 100644 --- a/drivers/char/mem.c +++ b/mm/char-mem.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * linux/drivers/char/mem.c + * mm/char-mem.c * * Copyright (C) 1991, 1992 Linus Torvalds * @@ -31,6 +31,8 @@ #include #include +#include "internal.h" + #define DEVMEM_MINOR 1 #define DEVPORT_MINOR 4 @@ -506,11 +508,7 @@ static int mmap_zero_prepare(struct vm_area_desc *desc) if (vma_desc_test(desc, VMA_MAYSHARE_BIT)) return shmem_zero_setup_desc(desc); - /* - * This is a highly unique situation where we mark a MAP_PRIVATE mapping - * of /dev/zero anonymous, despite it not being. - */ - vma_desc_set_anonymous(desc); + /* MAP_PRIVATE semantics are taken care of for us by core mm. */ return 0; } @@ -707,6 +705,17 @@ static const struct memdev { #endif }; +/** + * file_is_dev_zero() - is the specified @file associated with the /dev/zero + * driver? + * @file: File to test. + * Returns: true if it is, false otherwise. + */ +bool file_is_dev_zero(const struct file *file) +{ + return file && file->f_op == &zero_fops; +} + static int memory_open(struct inode *inode, struct file *filp) { int minor; diff --git a/mm/cma.h b/mm/cma.h index ab6d39898ea5..68e574b0f951 100644 --- a/mm/cma.h +++ b/mm/cma.h @@ -2,14 +2,24 @@ #ifndef __MM_CMA_H__ #define __MM_CMA_H__ +#include #include +#include +#include +#include + +#ifdef CONFIG_CMA_DEBUGFS #include +#endif + +#ifdef CONFIG_CMA_SYSFS #include struct cma_kobject { struct kobject kobj; struct cma *cma; }; +#endif /* * Multi-range support. This can be useful if the size of the allocation @@ -38,10 +48,10 @@ struct cma_memrange { #define CMA_MAX_RANGES 8 struct cma { - unsigned long count; - unsigned long available_count; + unsigned long count; + unsigned long available_count; unsigned int order_per_bit; /* Order of pages represented by one bit */ - spinlock_t lock; + spinlock_t lock; struct mutex alloc_mutex; #ifdef CONFIG_CMA_DEBUGFS struct hlist_head mem_head; @@ -87,10 +97,11 @@ void cma_sysfs_account_fail_pages(struct cma *cma, unsigned long nr_pages); void cma_sysfs_account_release_pages(struct cma *cma, unsigned long nr_pages); #else static inline void cma_sysfs_account_success_pages(struct cma *cma, - unsigned long nr_pages) {}; + unsigned long nr_pages) {} static inline void cma_sysfs_account_fail_pages(struct cma *cma, - unsigned long nr_pages) {}; + unsigned long nr_pages) {} static inline void cma_sysfs_account_release_pages(struct cma *cma, - unsigned long nr_pages) {}; + unsigned long nr_pages) {} #endif + #endif diff --git a/mm/compaction.c b/mm/compaction.c index a049415512c6..4994e200bbec 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -3197,7 +3197,6 @@ static int kcompactd(void *p) long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC); long timeout = default_timeout; - current->flags |= PF_KCOMPACTD; set_freezable(); pgdat->kcompactd_max_order = 0; @@ -3254,11 +3253,15 @@ static int kcompactd(void *p) pgdat->proactive_compact_trigger = false; } - current->flags &= ~PF_KCOMPACTD; - return 0; } +bool current_is_kcompactd(void) +{ + return kthread_func(current) == kcompactd; +} +EXPORT_SYMBOL_GPL(current_is_kcompactd); + /* * This kcompactd start function will be called by init and node-hot-add. * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added. diff --git a/mm/damon/core.c b/mm/damon/core.c index ab3c4d754964..733025b36745 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -10,11 +10,11 @@ #include #include #include +#include #include #include #include #include -#include /* for damon_get_folio() used by node eligible memory metrics */ #include "ops-common.h" @@ -111,6 +111,63 @@ int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id) return err; } +struct damon_prep *damon_new_prep(enum damon_prep_action action) +{ + struct damon_prep *prep; + + prep = kmalloc_obj(*prep); + if (!prep) + return NULL; + prep->action = action; + INIT_LIST_HEAD(&prep->list); + return prep; +} + +void damon_add_prep(struct damon_probe *p, struct damon_prep *prep) +{ + list_add_tail(&prep->list, &p->preps); +} + +static void damon_del_prep(struct damon_prep *p) +{ + list_del(&p->list); +} + +static void damon_free_prep(struct damon_prep *p) +{ + kfree(p); +} + +static void damon_destroy_prep(struct damon_prep *p) +{ + damon_del_prep(p); + damon_free_prep(p); +} + +static struct damon_prep *damon_nth_prep(int n, struct damon_probe *p) +{ + struct damon_prep *prep; + int i = 0; + + damon_for_each_prep(prep, p) { + if (i++ == n) + return prep; + } + return NULL; +} + +static bool damon_has_prep(struct damon_ctx *c) +{ + struct damon_prep *prep; + struct damon_probe *probe; + + damon_for_each_probe(probe, c) { + damon_for_each_prep(prep, probe) + return true; + } + return false; +} + struct damon_filter *damon_new_filter(enum damon_filter_type type, bool matching, bool allow) { @@ -167,6 +224,7 @@ struct damon_probe *damon_new_probe(void) if (!p) return NULL; p->weight = 0; + INIT_LIST_HEAD(&p->preps); INIT_LIST_HEAD(&p->filters); INIT_LIST_HEAD(&p->list); return p; @@ -184,8 +242,11 @@ static void damon_del_probe(struct damon_probe *p) static void damon_free_probe(struct damon_probe *p) { + struct damon_prep *prep, *prep_next; struct damon_filter *f, *next; + damon_for_each_prep_safe(prep, prep_next, p) + damon_free_prep(prep); damon_for_each_filter_safe(f, next, p) damon_free_filter(f); kfree(p); @@ -408,11 +469,12 @@ static bool damon_is_last_region(struct damon_region *r, * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region. * @r: region to get the weighted sum of. * @last: if the request is for last-window aggregated probe hits. + * @mv: use moving sum. * @ctx: context of &r. * * Return: the weighted sum of probe hits of the region. */ -unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, bool mv, struct damon_ctx *ctx) { struct damon_probe *probe; @@ -422,6 +484,9 @@ unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, damon_for_each_probe(probe, ctx) { if (last) sum += r->last_probe_hits[i++] * probe->weight; + else if (mv) + sum += damon_probe_hits_mvsum(i++, r, ctx) * + probe->weight; else sum += r->probe_hits[i++] * probe->weight; } @@ -593,6 +658,7 @@ bool damos_filter_for_ops(enum damos_filter_type type) switch (type) { case DAMOS_FILTER_TYPE_ADDR: case DAMOS_FILTER_TYPE_TARGET: + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: return false; default: break; @@ -635,6 +701,8 @@ struct damos_quota_goal *damos_new_quota_goal( return NULL; goal->metric = metric; goal->target_value = target_value; + if (metric == DAMOS_QUOTA_SOME_MEM_PSI_US) + goal->last_psi_total = U64_MAX; INIT_LIST_HEAD(&goal->list); return goal; } @@ -795,9 +863,15 @@ void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) list_add_tail(&t->list, &ctx->adaptive_targets); } -bool damon_targets_empty(struct damon_ctx *ctx) +/* + * Assign the struct pid of the given pid number to the given target. + */ +int damon_set_target_pid(struct damon_target *t, int pid) { - return list_empty(&ctx->adaptive_targets); + t->pid = find_get_pid(pid); + if (!t->pid) + return -EINVAL; + return 0; } static void damon_del_target(struct damon_target *t) @@ -864,6 +938,8 @@ struct damon_ctx *damon_new_ctx(void) INIT_LIST_HEAD(&ctx->adaptive_targets); INIT_LIST_HEAD(&ctx->schemes); + ctx->call_controls_obsolete = true; + ctx->walk_control_obsolete = true; prandom_seed_state(&ctx->rnd_state, get_random_u64()); return ctx; @@ -1122,6 +1198,9 @@ static void damos_commit_quota_goal_union( struct damos_quota_goal *dst, struct damos_quota_goal *src) { switch (dst->metric) { + case DAMOS_QUOTA_SOME_MEM_PSI_US: + dst->last_psi_total = U64_MAX; + break; case DAMOS_QUOTA_NODE_MEM_USED_BP: case DAMOS_QUOTA_NODE_MEM_FREE_BP: dst->nid = src->nid; @@ -1131,20 +1210,25 @@ static void damos_commit_quota_goal_union( dst->nid = src->nid; dst->memcg_id = src->memcg_id; break; + case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: + dst->nid = src->nid; + break; default: break; } } -static void damos_commit_quota_goal( +static int damos_commit_quota_goal( struct damos_quota_goal *dst, struct damos_quota_goal *src) { + if (!src->target_value) + return -EINVAL; dst->metric = src->metric; dst->target_value = src->target_value; if (dst->metric == DAMOS_QUOTA_USER_INPUT) dst->current_value = src->current_value; - /* keep last_psi_total as is, since it will be updated in next cycle */ damos_commit_quota_goal_union(dst, src); + return 0; } /** @@ -1162,14 +1246,17 @@ static void damos_commit_quota_goal( int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src) { struct damos_quota_goal *dst_goal, *next, *src_goal, *new_goal; - int i = 0, j = 0; + int i = 0, j = 0, err; damos_for_each_quota_goal_safe(dst_goal, next, dst) { src_goal = damos_nth_quota_goal(i++, src); - if (src_goal) - damos_commit_quota_goal(dst_goal, src_goal); - else + if (src_goal) { + err = damos_commit_quota_goal(dst_goal, src_goal); + if (err) + return err; + } else { damos_destroy_quota_goal(dst_goal); + } } damos_for_each_quota_goal_safe(src_goal, next, src) { if (j++ < i) @@ -1178,7 +1265,11 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src) src_goal->metric, src_goal->target_value); if (!new_goal) return -ENOMEM; - damos_commit_quota_goal(new_goal, src_goal); + err = damos_commit_quota_goal(new_goal, src_goal); + if (err) { + damos_free_quota_goal(new_goal); + return err; + } damos_add_quota_goal(dst, new_goal); } return 0; @@ -1227,7 +1318,7 @@ static struct damos_filter *damos_nth_ops_filter(int n, struct damos *s) return NULL; } -static void damos_commit_filter_arg( +static int damos_commit_filter_arg( struct damos_filter *dst, struct damos_filter *src) { switch (dst->type) { @@ -1235,39 +1326,53 @@ static void damos_commit_filter_arg( dst->memcg_id = src->memcg_id; break; case DAMOS_FILTER_TYPE_ADDR: + if (src->addr_range.end < src->addr_range.start) + return -EINVAL; dst->addr_range = src->addr_range; break; case DAMOS_FILTER_TYPE_TARGET: dst->target_idx = src->target_idx; break; case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: + if (src->sz_range.max < src->sz_range.min) + return -EINVAL; dst->sz_range = src->sz_range; break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + if (src->range_max < src->range_min) + return -EINVAL; + dst->range_min = src->range_min; + dst->range_max = src->range_max; + break; default: break; } + return 0; } -static void damos_commit_filter( +static int damos_commit_filter( struct damos_filter *dst, struct damos_filter *src) { dst->type = src->type; dst->matching = src->matching; dst->allow = src->allow; - damos_commit_filter_arg(dst, src); + return damos_commit_filter_arg(dst, src); } static int damos_commit_core_filters(struct damos *dst, struct damos *src) { struct damos_filter *dst_filter, *next, *src_filter, *new_filter; - int i = 0, j = 0; + int i = 0, j = 0, err; damos_for_each_core_filter_safe(dst_filter, next, dst) { src_filter = damos_nth_core_filter(i++, src); - if (src_filter) - damos_commit_filter(dst_filter, src_filter); - else + if (src_filter) { + err = damos_commit_filter(dst_filter, src_filter); + if (err) + return err; + } else { damos_destroy_filter(dst_filter); + } } damos_for_each_core_filter_safe(src_filter, next, src) { @@ -1279,7 +1384,11 @@ static int damos_commit_core_filters(struct damos *dst, struct damos *src) src_filter->allow); if (!new_filter) return -ENOMEM; - damos_commit_filter_arg(new_filter, src_filter); + err = damos_commit_filter_arg(new_filter, src_filter); + if (err) { + damos_destroy_filter(new_filter); + return err; + } damos_add_filter(dst, new_filter); } return 0; @@ -1288,14 +1397,17 @@ static int damos_commit_core_filters(struct damos *dst, struct damos *src) static int damos_commit_ops_filters(struct damos *dst, struct damos *src) { struct damos_filter *dst_filter, *next, *src_filter, *new_filter; - int i = 0, j = 0; + int i = 0, j = 0, err; damos_for_each_ops_filter_safe(dst_filter, next, dst) { src_filter = damos_nth_ops_filter(i++, src); - if (src_filter) - damos_commit_filter(dst_filter, src_filter); - else + if (src_filter) { + err = damos_commit_filter(dst_filter, src_filter); + if (err) + return err; + } else { damos_destroy_filter(dst_filter); + } } damos_for_each_ops_filter_safe(src_filter, next, src) { @@ -1307,7 +1419,11 @@ static int damos_commit_ops_filters(struct damos *dst, struct damos *src) src_filter->allow); if (!new_filter) return -ENOMEM; - damos_commit_filter_arg(new_filter, src_filter); + err = damos_commit_filter_arg(new_filter, src_filter); + if (err) { + damos_destroy_filter(new_filter); + return err; + } damos_add_filter(dst, new_filter); } return 0; @@ -1345,6 +1461,13 @@ static bool damon_valid_probe_params(struct damon_ctx *ctx) unsigned char max_probe_hits; struct damon_probe *probe; unsigned int wsum, wsum_to_add; + int nr_probes; + + nr_probes = 0; + damon_for_each_probe(probe, ctx) + nr_probes++; + if (nr_probes > DAMON_MAX_PROBES) + return false; if (!damon_has_probe_weights(ctx)) return true; @@ -1666,7 +1789,37 @@ out: return err; } -static void damon_commit_filter(struct damon_filter *dst, +static void damon_commit_prep(struct damon_prep *dst, struct damon_prep *src) +{ + dst->action = src->action; +} + +static int damon_commit_preps(struct damon_probe *dst, struct damon_probe *src) +{ + struct damon_prep *dst_prep, *next, *src_prep, *new_prep; + int i = 0, j = 0; + + damon_for_each_prep_safe(dst_prep, next, dst) { + src_prep = damon_nth_prep(i++, src); + if (src_prep) + damon_commit_prep(dst_prep, src_prep); + else + damon_destroy_prep(dst_prep); + } + + damon_for_each_prep_safe(src_prep, next, src) { + if (j++ < i) + continue; + + new_prep = damon_new_prep(src_prep->action); + if (!new_prep) + return -ENOMEM; + damon_add_prep(dst, new_prep); + } + return 0; +} + +static int damon_commit_filter(struct damon_filter *dst, struct damon_filter *src) { dst->type = src->type; @@ -1676,23 +1829,33 @@ static void damon_commit_filter(struct damon_filter *dst, case DAMON_FILTER_TYPE_MEMCG: dst->memcg_id = src->memcg_id; break; + case DAMON_FILTER_TYPE_HUGEPAGE_SIZE: + if (src->range_max < src->range_min) + return -EINVAL; + dst->range_min = src->range_min; + dst->range_max = src->range_max; + break; default: break; } + return 0; } static int damon_commit_filters(struct damon_probe *dst, struct damon_probe *src) { struct damon_filter *dst_filter, *next, *src_filter, *new_filter; - int i = 0, j = 0; + int i = 0, j = 0, err; damon_for_each_filter_safe(dst_filter, next, dst) { src_filter = damon_nth_filter(i++, src); - if (src_filter) - damon_commit_filter(dst_filter, src_filter); - else + if (src_filter) { + err = damon_commit_filter(dst_filter, src_filter); + if (err) + return err; + } else { damon_destroy_filter(dst_filter); + } } damon_for_each_filter_safe(src_filter, next, src) { @@ -1707,6 +1870,14 @@ static int damon_commit_filters(struct damon_probe *dst, case DAMON_FILTER_TYPE_MEMCG: new_filter->memcg_id = src_filter->memcg_id; break; + case DAMON_FILTER_TYPE_HUGEPAGE_SIZE: + if (src_filter->range_max < src_filter->range_min) { + damon_destroy_filter(new_filter); + return -EINVAL; + } + new_filter->range_min = src_filter->range_min; + new_filter->range_max = src_filter->range_max; + break; default: break; } @@ -1724,6 +1895,9 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) src_probe = damon_nth_probe(i++, src); if (src_probe) { dst_probe->weight = src_probe->weight; + err = damon_commit_preps(dst_probe, src_probe); + if (err) + return err; err = damon_commit_filters(dst_probe, src_probe); if (err) return err; @@ -1741,6 +1915,9 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) return -ENOMEM; damon_add_probe(dst, new_probe); new_probe->weight = src_probe->weight; + err = damon_commit_preps(new_probe, src_probe); + if (err) + return err; err = damon_commit_filters(new_probe, src_probe); if (err) return err; @@ -1852,20 +2029,6 @@ out: return err; } -/** - * damon_nr_running_ctxs() - Return number of currently running contexts. - */ -int damon_nr_running_ctxs(void) -{ - int nr_ctxs; - - mutex_lock(&damon_lock); - nr_ctxs = nr_running_ctxs; - mutex_unlock(&damon_lock); - - return nr_ctxs; -} - /* Returns the size upper limit for each monitoring region */ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) { @@ -1954,8 +2117,6 @@ static int __damon_start(struct damon_ctx *ctx) return err; } -static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src); - /** * damon_start() - Starts the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to start monitoring @@ -2095,16 +2256,17 @@ int damon_kdamond_pid(struct damon_ctx *ctx) * synchronization. The return value of the function will be saved in * &damon_call_control->return_code. * - * Note that this function should be called only after damon_start() with the - * @ctx has succeeded. Otherwise, this function could fall into an indefinite - * wait. - * * When this function is failed, the @ctx is guaranteed to be stopped. * + * This function should not be called in parallel to damon_start() for the + * @ctx. In the case, this function could indefinitely hang. + * * Return: 0 on success, negative error code otherwise. */ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) { + if (!ctx) + return -EINVAL; if (!control->repeat) init_completion(&control->completion); control->canceled = false; @@ -2150,10 +2312,6 @@ canceled: * passed at least one &damos->apply_interval_us, kdamond marks the request as * completed so that damos_walk() can wakeup and return. * - * Note that this function should be called only after damon_start() with the - * @ctx has succeeded. Otherwise, this function could fall into an indefinite - * wait. - * * Return: 0 on success, negative error code otherwise. */ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) @@ -2328,7 +2486,7 @@ static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r, * This function checks if a given region should be skipped or not for the * reason. If only the starting part of the region has previously charged, * this function splits the region into two so that the second one covers the - * area that not charged in the previous charge widnow, and return true. The + * area that not charged in the previous charge window, and return true. The * caller can see the second one on the next iteration of the region walk. * Note that this means the caller should use damon_for_each_region() instead * of damon_for_each_region_safe(). If damon_for_each_region_safe() is used, @@ -2396,7 +2554,7 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t, bool matched = false; struct damon_target *ti; int target_idx = 0; - unsigned long start, end; + unsigned long start, end, wsum; switch (filter->type) { case DAMOS_FILTER_TYPE_TARGET: @@ -2431,6 +2589,11 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t, damon_split_region_at(t, r, end - r->ar.start); matched = true; break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + wsum = damon_probe_hits_wsum(r, false, true, ctx); + matched = filter->range_min <= wsum && + wsum <= filter->range_max; + break; default: return false; } @@ -2489,7 +2652,7 @@ static void damos_walk_call_walk(struct damon_ctx *ctx, struct damon_target *t, * This function is called when kdamond finished applying the action of a DAMOS * scheme to all regions that eligible for the given &damos->apply_interval_us. * If every scheme of @ctx including @s now finished walking for at least one - * &damos->apply_interval_us, this function makrs the handling of the given + * &damos->apply_interval_us, this function marks the handling of the given * DAMOS walk request is done, so that damos_walk() can wake up and return. */ static void damos_walk_complete(struct damon_ctx *ctx, struct damos *s) @@ -2616,10 +2779,12 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, c->min_region_sz); if (!sz) goto update_stat; - damon_split_region_at(t, r, sz); + if (damon_split_region_at(t, r, sz)) + goto update_stat; } if (damos_core_filter_out(c, t, r, s)) return; + sz = damon_sz_region(r); ktime_get_coarse_ts64(&begin); trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses, damon_nr_regions(t), do_trace); @@ -2762,15 +2927,8 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input, if (score >= goal * 2) return min_input; - if (over_achieving) - score_goal_diff = score - goal; - else - score_goal_diff = goal - score; - - if (last_input < ULONG_MAX / score_goal_diff) - compensation = last_input * score_goal_diff / goal; - else - compensation = last_input / goal * score_goal_diff; + score_goal_diff = abs_diff(score, goal); + compensation = mult_frac(last_input, score_goal_diff, goal); if (over_achieving) return max(last_input - compensation, min_input); @@ -2798,6 +2956,28 @@ static inline u64 damos_get_some_mem_psi_total(void) #endif /* CONFIG_PSI */ +static void damos_set_psi_current_val(u64 now_psi_total, + struct damos_quota_goal *goal, struct damos *s) +{ + u64 last_psi_total = goal->last_psi_total; + + goal->last_psi_total = now_psi_total; + if (last_psi_total != U64_MAX) { + goal->current_value = now_psi_total - last_psi_total; + return; + } + /* uninitialized last_psi_total; make no effect this round */ + if (s->quota.goal_tuner == DAMOS_QUOTA_GOAL_TUNER_CONSIST) { + goal->current_value = goal->target_value; + return; + } + /* let temporal tuner show the same achievement as in the last round */ + if (!s->quota.esz) + goal->current_value = goal->target_value; + else + goal->current_value = 0; +} + #ifdef CONFIG_NUMA static bool invalid_mem_node(int nid) { @@ -2818,6 +2998,13 @@ static __kernel_ulong_t damos_get_node_mem_bp( } si_meminfo_node(&i, goal->nid); + if (!i.totalram || i.totalram < i.freeram) { + if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP) + return 10000; + else /* DAMOS_QUOTA_NODE_MEM_FREE_BP */ + return 0; + } + if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP) numerator = i.totalram - i.freeram; else /* DAMOS_QUOTA_NODE_MEM_FREE_BP */ @@ -2858,6 +3045,12 @@ static unsigned long damos_get_node_memcg_used_bp( mem_cgroup_put(memcg); si_meminfo_node(&i, goal->nid); + if (!i.totalram || i.totalram < used_pages) { + if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP) + return 10000; + else /* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */ + return 0; + } if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP) numerator = used_pages; else /* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */ @@ -3004,12 +3197,28 @@ static unsigned int damos_get_in_active_mem_bp(bool active_ratio) global_node_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE); inactive = global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_ANON) + global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE); - total = active + inactive; + total = max(active + inactive, 1); if (active_ratio) return mult_frac(active, 10000, total); return mult_frac(inactive, 10000, total); } +static unsigned int damos_hugepage_mem_bp(void) +{ + unsigned long thp, total_pages, free_pages; + + total_pages = totalram_pages(); + free_pages = global_zone_page_state(NR_FREE_PAGES); + + if (total_pages <= free_pages) + return 10000; + + thp = global_node_page_state(NR_ANON_THPS) + + global_node_page_state(NR_SHMEM_THPS) + + global_node_page_state(NR_FILE_THPS); + return mult_frac(thp, 10000, total_pages - free_pages); +} + static void damos_set_quota_goal_current_value(struct damon_ctx *c, struct damos *s, struct damos_quota_goal *goal) { @@ -3021,8 +3230,7 @@ static void damos_set_quota_goal_current_value(struct damon_ctx *c, break; case DAMOS_QUOTA_SOME_MEM_PSI_US: now_psi_total = damos_get_some_mem_psi_total(); - goal->current_value = now_psi_total - goal->last_psi_total; - goal->last_psi_total = now_psi_total; + damos_set_psi_current_val(now_psi_total, goal, s); break; case DAMOS_QUOTA_NODE_MEM_USED_BP: case DAMOS_QUOTA_NODE_MEM_FREE_BP: @@ -3041,6 +3249,9 @@ static void damos_set_quota_goal_current_value(struct damon_ctx *c, goal->current_value = damos_get_node_eligible_mem_bp(c, s, goal->nid); break; + case DAMOS_QUOTA_HUGEPAGE_MEM_BP: + goal->current_value = damos_hugepage_mem_bp(); + break; default: break; } @@ -3174,6 +3385,8 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) damos_trace_esz(c, s, quota); } + if (damos_quota_is_full(quota, c->min_region_sz)) + return; if (!c->ops.get_scheme_score) return; @@ -3308,7 +3521,7 @@ static unsigned int damon_merge_score(struct damon_region *r, bool last, struct damon_ctx *ctx, bool use_probe_hits) { if (use_probe_hits) - return damon_probe_hits_wsum(r, last, ctx); + return damon_probe_hits_wsum(r, last, false, ctx); if (last) return r->last_nr_accesses; return r->nr_accesses; @@ -3384,8 +3597,7 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, unsigned int max_thres; bool count_age = true; - max_thres = c->attrs.aggr_interval / - (c->attrs.sample_interval ? c->attrs.sample_interval : 1); + max_thres = damon_nr_samples_per_aggr(&c->attrs); while (true) { nr_regions = 0; damon_for_each_target(t, c) { @@ -3608,10 +3820,6 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) /* higher than high watermark or lower than low watermark */ if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { - if (scheme->wmarks.activated) - pr_debug("deactivate a scheme (%d) for %s wmark\n", - scheme->action, - str_high_low(metric > scheme->wmarks.high)); scheme->wmarks.activated = false; return scheme->wmarks.interval; } @@ -3621,8 +3829,6 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) !scheme->wmarks.activated) return scheme->wmarks.interval; - if (!scheme->wmarks.activated) - pr_debug("activate a scheme (%d)\n", scheme->action); scheme->wmarks.activated = true; return 0; } @@ -3765,8 +3971,6 @@ static int kdamond_fn(void *data) struct damon_ctx *ctx = data; unsigned long sz_limit = 0; - pr_debug("kdamond (%d) starts\n", current->pid); - mutex_lock(&ctx->call_controls_lock); ctx->call_controls_obsolete = false; mutex_unlock(&ctx->call_controls_lock); @@ -3796,14 +4000,19 @@ static int kdamond_fn(void *data) unsigned long next_ops_update_sis = ctx->next_ops_update_sis; unsigned long sample_interval = ctx->attrs.sample_interval; bool access_check_disabled = damon_has_probe_weights(ctx); + bool do_prep; unsigned int max_merge_score = 0, max_wsum; bool get_max_wsum; if (kdamond_wait_activation(ctx)) break; + do_prep = ctx->ops.prep_probes && damon_has_prep(ctx); + if (!access_check_disabled && ctx->ops.prepare_access_checks) ctx->ops.prepare_access_checks(ctx); + if (do_prep) + ctx->ops.prep_probes(ctx, access_check_disabled); kdamond_usleep(sample_interval); ctx->passed_sample_intervals++; @@ -3818,7 +4027,8 @@ static int kdamond_fn(void *data) else get_max_wsum = false; max_wsum = ctx->ops.apply_probes(ctx, - access_check_disabled, get_max_wsum); + access_check_disabled && !do_prep, + get_max_wsum); if (get_max_wsum) max_merge_score = max_wsum; } @@ -3914,7 +4124,6 @@ done: mutex_unlock(&ctx->walk_control_lock); damos_walk_cancel(ctx); - pr_debug("kdamond (%d) finishes\n", current->pid); mutex_lock(&ctx->kdamond_lock); ctx->kdamond = NULL; mutex_unlock(&ctx->kdamond_lock); @@ -3985,7 +4194,7 @@ static bool damon_find_system_rams_range(unsigned long *start, * This function sets the region of @t as requested by @start and @end. If the * values of @start and @end are zero, however, this function finds 'System * RAM' resources and sets the region to cover all the resource. In the latter - * case, this function saves the start and the end addresseses of the first and + * case, this function saves the start and the end addresses of the first and * the last resources in @start and @end, respectively. * * Return: 0 on success, negative error code otherwise. diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index bd847829a990..273efa3c913e 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -233,8 +233,6 @@ static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme, if (!active_mem_bp) return 0; - if (10000 < active_mem_bp) - return -EINVAL; goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp); if (!goal) return -ENOMEM; @@ -262,7 +260,7 @@ static int damon_lru_sort_add_filters(struct damos *hot_scheme, return -ENOMEM; damos_add_filter(hot_scheme, filter); - /* disabllow de-prioritizing young pages */ + /* disallow de-prioritizing young pages */ filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, false); if (!filter) return -ENOMEM; @@ -346,8 +344,6 @@ static int damon_lru_sort_commit_inputs_fn(void *arg) return damon_lru_sort_apply_parameters(); } -static bool damon_lru_sort_damon_has_started; - static int damon_lru_sort_commit_inputs_store(const char *val, const struct kernel_param *kp) { @@ -368,10 +364,6 @@ static int damon_lru_sort_commit_inputs_store(const char *val, if (!commit_inputs_request) return 0; - /* Skip damon_call() if ctx has not successfully started. */ - if (!damon_lru_sort_damon_has_started) - return -EINVAL; - err = damon_call(ctx, &control); return err ? err : control.return_code; @@ -422,8 +414,6 @@ static int damon_lru_sort_turn(bool on) err = damon_start(&ctx, 1, true); if (err) return err; - if (!damon_lru_sort_damon_has_started) - damon_lru_sort_damon_has_started = true; return damon_call(ctx, &call_control); } diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 8fc61d06d358..77366f42b3e5 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -3,6 +3,7 @@ * Common Code for Data Access Monitoring */ +#include #include #include #include @@ -14,14 +15,20 @@ #include "../internal.h" #include "ops-common.h" +static bool damon_folio_acceptable(struct folio *folio, bool monitor) +{ + return folio_test_lru(folio) || + (monitor && folio_test_hugetlb(folio)); +} + /* - * Get an online page for a pfn if it's in the LRU list. Otherwise, returns - * NULL. + * Get an online page for a pfn if it's in the LRU list, or a hugetlb folio if + * @monitor is set. Otherwise, returns NULL. * * The body of this function is stolen from the 'page_idle_get_folio()'. We * steal rather than reuse it because the code is quite simple. */ -struct folio *damon_get_folio(unsigned long pfn) +static struct folio *__damon_get_folio(unsigned long pfn, bool monitor) { struct page *page = pfn_to_online_page(pfn); struct folio *folio; @@ -32,13 +39,24 @@ struct folio *damon_get_folio(unsigned long pfn) folio = page_folio(page); if (!folio_try_get(folio)) return NULL; - if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) { + if (unlikely(page_folio(page) != folio) || + !damon_folio_acceptable(folio, monitor)) { folio_put(folio); folio = NULL; } return folio; } +struct folio *damon_get_folio(unsigned long pfn) +{ + return __damon_get_folio(pfn, false); +} + +struct folio *damon_get_monitor_folio(unsigned long pfn) +{ + return __damon_get_folio(pfn, true); +} + void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr) { pte_t pteval = ptep_get(pte); @@ -103,6 +121,42 @@ void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ } +#ifdef CONFIG_HUGETLB_PAGE +static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, pte_t *entry) +{ + unsigned long psize = huge_page_size(hstate_vma(vma)); + + if (!pte_young(*entry)) + return false; + *entry = huge_ptep_get_and_clear(mm, addr, pte, psize); + *entry = pte_mkold(*entry); + set_huge_pte_at(mm, addr, pte, *entry, psize); + return true; +} + +void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + bool referenced = false; + pte_t entry = huge_ptep_get(mm, addr, pte); + struct folio *folio = pfn_folio(pte_pfn(entry)); + + folio_get(folio); + + referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry); + if (mmu_notifier_clear_young(mm, addr, + addr + huge_page_size(hstate_vma(vma)))) + referenced = true; + + if (referenced) + folio_set_young(folio); + + folio_set_idle(folio); + folio_put(folio); +} +#endif /* CONFIG_HUGETLB_PAGE */ + #define DAMON_MAX_SUBSCORE (100) #define DAMON_MAX_AGE_IN_LOG (32) @@ -168,10 +222,15 @@ static bool damon_folio_mkold_one(struct folio *folio, while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; - if (pvmw.pte) - damon_ptep_mkold(pvmw.pte, vma, addr); - else + if (pvmw.pte) { + if (folio_test_hugetlb(folio)) + damon_hugetlb_mkold(pvmw.pte, vma->vm_mm, vma, + addr); + else + damon_ptep_mkold(pvmw.pte, vma, addr); + } else { damon_pmdp_mkold(pvmw.pmd, vma, addr); + } } return true; } @@ -196,27 +255,55 @@ void damon_folio_mkold(struct folio *folio) } +#ifdef CONFIG_HUGETLB_PAGE +static bool damon_hugetlb_young(pte_t *pte, struct vm_area_struct *vma, + unsigned long addr, struct folio *folio) +{ + pte_t entry = huge_ptep_get(vma->vm_mm, addr, pte); + + return (pte_present(entry) && pte_young(entry)) || + !folio_test_idle(folio) || + mmu_notifier_test_young(vma->vm_mm, addr); +} +#else +static bool damon_hugetlb_young(pte_t *pte, struct vm_area_struct *vma, + unsigned long addr, struct folio *folio) +{ + return false; +} +#endif /* CONFIG_HUGETLB_PAGE */ + +static bool damon_pte_young(pte_t *pte, struct vm_area_struct *vma, + unsigned long addr, struct folio *folio) +{ + pte_t entry = ptep_get(pte); + + /* + * PFN swap PTEs, such as device-exclusive ones, that actually map + * pages are "old" from a CPU perspective. The MMU notifier takes care + * of any device aspects. + */ + return (pte_present(entry) && pte_young(entry)) || + !folio_test_idle(folio) || + mmu_notifier_test_young(vma->vm_mm, addr); +} + static bool damon_folio_young_one(struct folio *folio, struct vm_area_struct *vma, unsigned long addr, void *arg) { bool *accessed = arg; DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); - pte_t pte; *accessed = false; while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; if (pvmw.pte) { - pte = ptep_get(pvmw.pte); - - /* - * PFN swap PTEs, such as device-exclusive ones, that - * actually map pages are "old" from a CPU perspective. - * The MMU notifier takes care of any device aspects. - */ - *accessed = (pte_present(pte) && pte_young(pte)) || - !folio_test_idle(folio) || - mmu_notifier_test_young(vma->vm_mm, addr); + if (folio_test_hugetlb(folio)) + *accessed = damon_hugetlb_young(pvmw.pte, vma, + addr, folio); + else + *accessed = damon_pte_young(pvmw.pte, vma, + addr, folio); } else { #ifdef CONFIG_TRANSPARENT_HUGEPAGE pmd_t pmd = pmdp_get(pvmw.pmd); @@ -335,6 +422,19 @@ static unsigned int __damon_migrate_folio_list( return nr_succeeded; } +static void damon_putback_folio_list(struct list_head *folio_list) +{ + struct folio *folio; + + while (!list_empty(folio_list)) { + folio = lru_to_folio(folio_list); + list_del(&folio->lru); + node_stat_sub_folio(folio, NR_ISOLATED_ANON + + folio_is_file_lru(folio)); + folio_putback_lru(folio); + } +} + static unsigned int damon_migrate_folio_list(struct list_head *folio_list, struct pglist_data *pgdat, int target_nid) @@ -376,13 +476,7 @@ keep: list_splice(&ret_folios, folio_list); - while (!list_empty(folio_list)) { - folio = lru_to_folio(folio_list); - list_del(&folio->lru); - node_stat_sub_folio(folio, NR_ISOLATED_ANON + - folio_is_file_lru(folio)); - folio_putback_lru(folio); - } + damon_putback_folio_list(folio_list); return nr_migrated; } @@ -399,14 +493,7 @@ unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid) if (target_nid < 0 || target_nid >= MAX_NUMNODES || !node_state(target_nid, N_MEMORY)) { - while (!list_empty(folio_list)) { - struct folio *folio = lru_to_folio(folio_list); - - list_del(&folio->lru); - node_stat_sub_folio(folio, NR_ISOLATED_ANON + - folio_is_file_lru(folio)); - folio_putback_lru(folio); - } + damon_putback_folio_list(folio_list); return nr_migrated; } @@ -444,3 +531,45 @@ bool damos_ops_has_filter(struct damos *s) return true; return false; } + +bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio) +{ + bool matched = false; + struct mem_cgroup *memcg; + size_t folio_sz; + + switch (filter->type) { + case DAMON_FILTER_TYPE_ANON: + if (!folio) { + matched = false; + break; + } + matched = folio_test_anon(folio); + break; + case DAMON_FILTER_TYPE_MEMCG: + if (!folio) { + matched = false; + break; + } + rcu_read_lock(); + memcg = folio_memcg_check(folio); + if (!memcg) + matched = false; + else + matched = filter->memcg_id == mem_cgroup_id(memcg); + rcu_read_unlock(); + break; + case DAMON_FILTER_TYPE_HUGEPAGE_SIZE: + if (!folio) { + matched = false; + break; + } + folio_sz = folio_size(folio); + matched = filter->range_min <= folio_sz && + folio_sz <= filter->range_max; + break; + default: + break; + } + return matched == filter->matching; +} diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index 38d295488fa1..ee2705825100 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -6,9 +6,19 @@ #include struct folio *damon_get_folio(unsigned long pfn); +struct folio *damon_get_monitor_folio(unsigned long pfn); void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr); void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr); +#ifdef CONFIG_HUGETLB_PAGE +void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr); +#else +static inline void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ +} +#endif /* CONFIG_HUGETLB_PAGE */ void damon_folio_mkold(struct folio *folio); bool damon_folio_young(struct folio *folio); @@ -21,3 +31,5 @@ bool damos_folio_filter_match(struct damos_filter *filter, struct folio *folio); unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid); bool damos_ops_has_filter(struct damos *s); + +bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5c6c3a597fd0..2cfdc356b415 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -37,7 +37,7 @@ static unsigned long damon_pa_core_addr( static void damon_pa_mkold(phys_addr_t paddr) { - struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); + struct folio *folio = damon_get_monitor_folio(PHYS_PFN(paddr)); if (!folio) return; @@ -67,7 +67,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) static bool damon_pa_young(phys_addr_t paddr) { - struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); + struct folio *folio = damon_get_monitor_folio(PHYS_PFN(paddr)); bool accessed; if (!folio) @@ -105,41 +105,65 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) return max_nr_accesses; } +static void damon_pa_prep_probes_region(struct damon_region *r, + struct damon_probe *probe, struct damon_ctx *ctx) +{ + struct damon_prep *p; + + damon_for_each_prep(p, probe) { + switch (p->action) { + case DAMON_PREP_SET_PGIDLE: + damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, + ctx->addr_unit)); + break; + default: + break; + } + } +} + +static void damon_pa_prep_probes(struct damon_ctx *ctx, bool set_samples) +{ + struct damon_target *t; + struct damon_region *r; + struct damon_probe *p; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) { + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); + damon_for_each_probe(p, ctx) + damon_pa_prep_probes_region(r, p, ctx); + } + } +} + static bool damon_pa_filter_match(struct damon_filter *filter, struct folio *folio) { bool matched = false; - struct mem_cgroup *memcg; switch (filter->type) { - case DAMON_FILTER_TYPE_ANON: - if (!folio) { - matched = false; - break; - } - matched = folio_test_anon(folio); - break; - case DAMON_FILTER_TYPE_MEMCG: - if (!folio) { - matched = false; - break; - } - rcu_read_lock(); - memcg = folio_memcg_check(folio); - if (!memcg) + case DAMON_FILTER_TYPE_PGIDLE_UNSET: + if (!folio) matched = false; else - matched = filter->memcg_id == mem_cgroup_id(memcg); - rcu_read_unlock(); + matched = damon_folio_young(folio); + break; + case DAMON_FILTER_TYPE_PGIDLE_SET: + if (!folio) + matched = false; + else + matched = damon_folio_young(folio) == false; break; default: - break; + return damon_ops_filter_match(filter, folio); } return matched == filter->matching; } -static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, - struct damon_probe *p) +static bool damon_pa_filter_pass(struct folio *folio, struct damon_probe *p) { struct damon_filter *f; bool pass = true; @@ -175,7 +199,7 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, ctx->addr_unit); folio = damon_get_folio(PHYS_PFN(pa)); damon_for_each_probe(p, ctx) { - if (damon_pa_filter_pass(pa, folio, p)) + if (damon_pa_filter_pass(folio, p)) r->probe_hits[i]++; i++; } @@ -183,7 +207,7 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, folio_put(folio); if (return_max_wsum) max_wsum = max(damon_probe_hits_wsum(r, false, - ctx), max_wsum); + false, ctx), max_wsum); } } return max_wsum; @@ -264,7 +288,7 @@ static unsigned long damon_pa_pageout(struct damon_region *r, else list_add(&folio->lru, &folio_list); put_folio: - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } if (install_young_filter) @@ -302,7 +326,7 @@ static inline unsigned long damon_pa_de_activate( folio_deactivate(folio); applied += folio_nr_pages(folio); put_folio: - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } s->last_applied = folio; @@ -350,7 +374,7 @@ static unsigned long damon_pa_migrate(struct damon_region *r, folio_is_file_lru(folio)); list_add(&folio->lru, &folio_list); put_folio: - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } applied = damon_migrate_pages(&folio_list, s->target_nid); @@ -379,7 +403,7 @@ static unsigned long damon_pa_stat(struct damon_region *r, if (!damos_pa_filter_out(s, folio)) *sz_filter_passed += folio_size(folio) / addr_unit; - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } s->last_applied = folio; @@ -442,6 +466,7 @@ static int __init damon_pa_initcall(void) .update = NULL, .prepare_access_checks = damon_pa_prepare_access_checks, .check_accesses = damon_pa_check_accesses, + .prep_probes = damon_pa_prep_probes, .apply_probes = damon_pa_apply_probes, .target_valid = NULL, .apply_scheme = damon_pa_apply_scheme, diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 45d5557cc575..42a2c9cb1343 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -271,8 +271,6 @@ static int damon_reclaim_commit_inputs_fn(void *arg) return damon_reclaim_apply_parameters(); } -static bool damon_reclaim_damon_has_started; - static int damon_reclaim_commit_inputs_store(const char *val, const struct kernel_param *kp) { @@ -293,10 +291,6 @@ static int damon_reclaim_commit_inputs_store(const char *val, if (!commit_inputs_request) return 0; - /* Skip damon_call() if ctx has not successfully started. */ - if (!damon_reclaim_damon_has_started) - return -EINVAL; - err = damon_call(ctx, &control); return err ? err : control.return_code; @@ -343,8 +337,6 @@ static int damon_reclaim_turn(bool on) err = damon_start(&ctx, 1, true); if (err) return err; - if (!damon_reclaim_damon_has_started) - damon_reclaim_damon_has_started = true; return damon_call(ctx, &call_control); } diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 32f495a96b17..bfb6f0bc3f21 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -534,7 +534,8 @@ struct damon_sysfs_scheme_filter { bool allow; char *memcg_path; struct damon_addr_range addr_range; - struct damon_size_range sz_range; + unsigned long range_min; + unsigned long range_max; int target_idx; }; @@ -588,6 +589,10 @@ damos_sysfs_filter_type_names[] = { .type = DAMOS_FILTER_TYPE_TARGET, .name = "target", }, + { + .type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, + .name = "probe_hits_wsum", + }, }; static ssize_t type_show(struct kobject *kobj, @@ -778,7 +783,7 @@ static ssize_t min_show(struct kobject *kobj, struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - return sysfs_emit(buf, "%lu\n", filter->sz_range.min); + return sysfs_emit(buf, "%lu\n", filter->range_min); } static ssize_t min_store(struct kobject *kobj, @@ -786,7 +791,7 @@ static ssize_t min_store(struct kobject *kobj, { struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - int err = kstrtoul(buf, 0, &filter->sz_range.min); + int err = kstrtoul(buf, 0, &filter->range_min); return err ? err : count; } @@ -797,7 +802,7 @@ static ssize_t max_show(struct kobject *kobj, struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - return sysfs_emit(buf, "%lu\n", filter->sz_range.max); + return sysfs_emit(buf, "%lu\n", filter->range_max); } static ssize_t max_store(struct kobject *kobj, @@ -805,7 +810,7 @@ static ssize_t max_store(struct kobject *kobj, { struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - int err = kstrtoul(buf, 0, &filter->sz_range.max); + int err = kstrtoul(buf, 0, &filter->range_max); return err ? err : count; } @@ -1269,6 +1274,10 @@ struct damos_sysfs_qgoal_metric_name damos_sysfs_qgoal_metric_names[] = { .metric = DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, .name = "node_eligible_mem_bp", }, + { + .metric = DAMOS_QUOTA_HUGEPAGE_MEM_BP, + .name = "hugepage_mem_bp", + }, }; static ssize_t target_metric_show(struct kobject *kobj, @@ -2813,7 +2822,8 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, if (!filter) return -ENOMEM; - if (filter->type == DAMOS_FILTER_TYPE_MEMCG) { + switch (filter->type) { + case DAMOS_FILTER_TYPE_MEMCG: err = damon_sysfs_memcg_path_to_id( sysfs_filter->memcg_path, &filter->memcg_id); @@ -2821,22 +2831,23 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, damos_destroy_filter(filter); return err; } - } else if (filter->type == DAMOS_FILTER_TYPE_ADDR) { - if (sysfs_filter->addr_range.end < - sysfs_filter->addr_range.start) { - damos_destroy_filter(filter); - return -EINVAL; - } + break; + case DAMOS_FILTER_TYPE_ADDR: filter->addr_range = sysfs_filter->addr_range; - } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) { + break; + case DAMOS_FILTER_TYPE_TARGET: filter->target_idx = sysfs_filter->target_idx; - } else if (filter->type == DAMOS_FILTER_TYPE_HUGEPAGE_SIZE) { - if (sysfs_filter->sz_range.min > - sysfs_filter->sz_range.max) { - damos_destroy_filter(filter); - return -EINVAL; - } - filter->sz_range = sysfs_filter->sz_range; + break; + case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: + filter->sz_range.min = sysfs_filter->range_min; + filter->sz_range.max = sysfs_filter->range_max; + break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + filter->range_min = sysfs_filter->range_min; + filter->range_max = sysfs_filter->range_max; + break; + default: + break; } damos_add_filter(scheme, filter); diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index e3858ffab4b2..70a4b64fb8ea 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -3,7 +3,6 @@ * DAMON sysfs Interface */ -#include #include #include @@ -750,6 +749,222 @@ static const struct kobj_type damon_sysfs_intervals_ktype = { .default_groups = damon_sysfs_intervals_groups, }; +/* + * prep directory + */ + +struct damon_sysfs_prep { + struct kobject kobj; + enum damon_prep_action action; +}; + +static struct damon_sysfs_prep *damon_sysfs_prep_alloc(void) +{ + struct damon_sysfs_prep *prep; + + prep = kzalloc_obj(struct damon_sysfs_prep); + if (!prep) + return prep; + prep->action = DAMON_PREP_SET_PGIDLE; + return prep; +} + +struct damon_sysfs_prep_action_name { + const enum damon_prep_action action; + const char *name; +}; + +static const struct damon_sysfs_prep_action_name +damon_sysfs_prep_action_names[] = { + { + .action = DAMON_PREP_SET_PGIDLE, + .name = "set_pgidle", + }, +}; + +static ssize_t prep_action_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_prep *prep = container_of(kobj, + struct damon_sysfs_prep, kobj); + int i; + + for (i = 0; i < ARRAY_SIZE(damon_sysfs_prep_action_names); i++) { + const struct damon_sysfs_prep_action_name *action_name; + + action_name = &damon_sysfs_prep_action_names[i]; + if (action_name->action == prep->action) + return sysfs_emit(buf, "%s\n", action_name->name); + } + return -EINVAL; +} + +static ssize_t prep_action_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_prep *prep = container_of(kobj, + struct damon_sysfs_prep, kobj); + ssize_t ret = -EINVAL; + int i; + + for (i = 0; i < ARRAY_SIZE(damon_sysfs_prep_action_names); i++) { + const struct damon_sysfs_prep_action_name *action_name; + + action_name = &damon_sysfs_prep_action_names[i]; + if (sysfs_streq(buf, action_name->name)) { + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; + prep->action = action_name->action; + mutex_unlock(&damon_sysfs_lock); + ret = count; + break; + } + } + return ret; +} + +static void damon_sysfs_prep_release(struct kobject *kobj) +{ + struct damon_sysfs_prep *prep = container_of(kobj, + struct damon_sysfs_prep, kobj); + + kfree(prep); +} + +static struct kobj_attribute damon_sysfs_prep_prep_action_attr = + __ATTR_RW_MODE(prep_action, 0600); + +static struct attribute *damon_sysfs_prep_attrs[] = { + &damon_sysfs_prep_prep_action_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(damon_sysfs_prep); + +static const struct kobj_type damon_sysfs_prep_ktype = { + .release = damon_sysfs_prep_release, + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = damon_sysfs_prep_groups, +}; + +/* + * preps directory + */ + +struct damon_sysfs_preps { + struct kobject kobj; + struct damon_sysfs_prep **preps_arr; + int nr; +}; + +static struct damon_sysfs_preps *damon_sysfs_preps_alloc(void) +{ + return kzalloc_obj(struct damon_sysfs_preps); +} + +static void damon_sysfs_preps_rm_dirs(struct damon_sysfs_preps *preps) +{ + struct damon_sysfs_prep **preps_arr = preps->preps_arr; + int i; + + for (i = 0; i < preps->nr; i++) { + kobject_del(&preps_arr[i]->kobj); + kobject_put(&preps_arr[i]->kobj); + } + preps->nr = 0; + kfree(preps_arr); + preps->preps_arr = NULL; +} + +static int damon_sysfs_preps_add_dirs(struct damon_sysfs_preps *preps, + int nr_preps) +{ + struct damon_sysfs_prep **preps_arr, *prep; + int err, i; + + damon_sysfs_preps_rm_dirs(preps); + if (!nr_preps) + return 0; + + preps_arr = kmalloc_objs(*preps_arr, nr_preps, + GFP_KERNEL | __GFP_NOWARN); + if (!preps_arr) + return -ENOMEM; + preps->preps_arr = preps_arr; + + for (i = 0; i < nr_preps; i++) { + prep = damon_sysfs_prep_alloc(); + if (!prep) { + damon_sysfs_preps_rm_dirs(preps); + return -ENOMEM; + } + + err = kobject_init_and_add(&prep->kobj, + &damon_sysfs_prep_ktype, &preps->kobj, "%d", + i); + if (err) { + kobject_put(&prep->kobj); + damon_sysfs_preps_rm_dirs(preps); + return err; + } + + preps_arr[i] = prep; + preps->nr++; + } + return 0; +} + +static ssize_t nr_preps_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct damon_sysfs_preps *preps = container_of(kobj, + struct damon_sysfs_preps, kobj); + + return sysfs_emit(buf, "%d\n", preps->nr); +} + +static ssize_t nr_preps_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_preps *preps; + int nr, err = kstrtoint(buf, 0, &nr); + + if (err) + return err; + if (nr < 0) + return -EINVAL; + + preps = container_of(kobj, struct damon_sysfs_preps, kobj); + + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; + err = damon_sysfs_preps_add_dirs(preps, nr); + mutex_unlock(&damon_sysfs_lock); + if (err) + return err; + + return count; +} + +static void damon_sysfs_preps_release(struct kobject *kobj) +{ + kfree(container_of(kobj, struct damon_sysfs_preps, kobj)); +} + +static struct kobj_attribute damon_sysfs_preps_nr_attr = + __ATTR_RW_MODE(nr_preps, 0600); + +static struct attribute *damon_sysfs_preps_attrs[] = { + &damon_sysfs_preps_nr_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(damon_sysfs_preps); + +static const struct kobj_type damon_sysfs_preps_ktype = { + .release = damon_sysfs_preps_release, + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = damon_sysfs_preps_groups, +}; + /* * filter directory */ @@ -760,6 +975,8 @@ struct damon_sysfs_filter { bool matching; bool allow; char *path; + unsigned long range_min; + unsigned long range_max; }; static struct damon_sysfs_filter *damon_sysfs_filter_alloc(void) @@ -782,6 +999,18 @@ damon_sysfs_filter_type_names[] = { .type = DAMON_FILTER_TYPE_MEMCG, .name = "memcg", }, + { + .type = DAMON_FILTER_TYPE_PGIDLE_UNSET, + .name = "pgidle_unset", + }, + { + .type = DAMON_FILTER_TYPE_PGIDLE_SET, + .name = "pgidle_set", + }, + { + .type = DAMON_FILTER_TYPE_HUGEPAGE_SIZE, + .name = "hugepage_size", + }, }; static ssize_t type_show(struct kobject *kobj, @@ -904,6 +1133,44 @@ static ssize_t path_store(struct kobject *kobj, return count; } +static ssize_t min_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + + return sysfs_emit(buf, "%lu\n", filter->range_min); +} + +static ssize_t min_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + int err = kstrtoul(buf, 0, &filter->range_min); + + return err ? err : count; +} + +static ssize_t max_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + + return sysfs_emit(buf, "%lu\n", filter->range_max); +} + +static ssize_t max_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + int err = kstrtoul(buf, 0, &filter->range_max); + + return err ? err : count; +} + static void damon_sysfs_filter_release(struct kobject *kobj) { struct damon_sysfs_filter *filter = container_of(kobj, @@ -925,11 +1192,19 @@ static struct kobj_attribute damon_sysfs_filter_allow_attr = static struct kobj_attribute damon_sysfs_filter_path_attr = __ATTR_RW_MODE(path, 0600); +static struct kobj_attribute damon_sysfs_filter_min_attr = + __ATTR_RW_MODE(min, 0600); + +static struct kobj_attribute damon_sysfs_filter_max_attr = + __ATTR_RW_MODE(max, 0600); + static struct attribute *damon_sysfs_filter_attrs[] = { &damon_sysfs_filter_type_attr.attr, &damon_sysfs_filter_matching_attr.attr, &damon_sysfs_filter_allow_attr.attr, &damon_sysfs_filter_path_attr.attr, + &damon_sysfs_filter_min_attr.attr, + &damon_sysfs_filter_max_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_filter); @@ -1066,6 +1341,7 @@ static const struct kobj_type damon_sysfs_filters_ktype = { struct damon_sysfs_probe { struct kobject kobj; unsigned int weight; + struct damon_sysfs_preps *preps; struct damon_sysfs_filters *filters; }; @@ -1076,25 +1352,50 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void) static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe) { + struct damon_sysfs_preps *preps; struct damon_sysfs_filters *filters; int err; - filters = damon_sysfs_filters_alloc(); - if (!filters) + preps = damon_sysfs_preps_alloc(); + if (!preps) return -ENOMEM; + probe->preps = preps; + + err = kobject_init_and_add(&preps->kobj, &damon_sysfs_preps_ktype, + &probe->kobj, "preps"); + if (err) + goto put_preps_out; + + filters = damon_sysfs_filters_alloc(); + if (!filters) { + err = -ENOMEM; + goto del_preps_out; + } probe->filters = filters; err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype, &probe->kobj, "filters"); - if (err) { - kobject_put(&filters->kobj); - probe->filters = NULL; - } + if (err) + goto put_filters_out; + return err; + +put_filters_out: + kobject_put(&filters->kobj); + probe->filters = NULL; +del_preps_out: + kobject_del(&preps->kobj); +put_preps_out: + kobject_put(&preps->kobj); + probe->preps = NULL; return err; } static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) { + if (probe->preps) { + damon_sysfs_preps_rm_dirs(probe->preps); + kobject_put(&probe->preps->kobj); + } if (probe->filters) { damon_sysfs_filters_rm_dirs(probe->filters); kobject_put(&probe->filters->kobj); @@ -1234,7 +1535,7 @@ static ssize_t nr_probes_store(struct kobject *kobj, if (err) return err; - if (nr < 0 || nr > DAMON_MAX_PROBES) + if (nr < 0) return -EINVAL; probes = container_of(kobj, struct damon_sysfs_probes, kobj); @@ -1789,6 +2090,7 @@ struct damon_sysfs_kdamond { struct damon_sysfs_contexts *contexts; struct damon_ctx *damon_ctx; unsigned int refresh_ms; + unsigned long next_refresh_jiffies; }; static struct damon_sysfs_kdamond *damon_sysfs_kdamond_alloc(void) @@ -1933,6 +2235,23 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, return damon_set_attrs(ctx, &attrs); } +static int damon_sysfs_set_preps(struct damon_probe *probe, + struct damon_sysfs_preps *sys_preps) +{ + int i; + + for (i = 0; i < sys_preps->nr; i++) { + struct damon_sysfs_prep *sys_prep = sys_preps->preps_arr[i]; + struct damon_prep *prep; + + prep = damon_new_prep(sys_prep->action); + if (!prep) + return -ENOMEM; + damon_add_prep(probe, prep); + } + return 0; +} + static int damon_sysfs_set_filters(struct damon_probe *probe, struct damon_sysfs_filters *sys_filters) { @@ -1958,6 +2277,9 @@ static int damon_sysfs_set_filters(struct damon_probe *probe, damon_destroy_filter(filter); return err; } + } else if (filter->type == DAMON_FILTER_TYPE_HUGEPAGE_SIZE) { + filter->range_min = sys_filter->range_min; + filter->range_max = sys_filter->range_max; } damon_add_filter(probe, filter); } @@ -1968,7 +2290,15 @@ static int damon_sysfs_set_probe(struct damon_probe *probe, struct damon_sysfs_probe *sys_probe) { struct damon_sysfs_filters *sys_filters; + struct damon_sysfs_preps *sys_preps; + int err; + sys_preps = sys_probe->preps; + if (sys_preps) { + err = damon_sysfs_set_preps(probe, sys_preps); + if (err) + return err; + } sys_filters = sys_probe->filters; if (!sys_filters) return 0; @@ -2035,9 +2365,8 @@ static int damon_sysfs_add_target(struct damon_sysfs_target *sys_target, return -ENOMEM; damon_add_target(ctx, t); if (damon_target_has_pid(ctx)) { - t->pid = find_get_pid(sys_target->pid); - if (!t->pid) - /* caller will destroy targets */ + /* caller will destroy targets */ + if (damon_set_target_pid(t, sys_target->pid)) return -EINVAL; } t->obsolete = sys_target->obsolete; @@ -2215,17 +2544,15 @@ static struct damon_ctx *damon_sysfs_build_ctx( return ctx; } -static unsigned long damon_sysfs_next_update_jiffies; - static int damon_sysfs_repeat_call_fn(void *data) { struct damon_sysfs_kdamond *sysfs_kdamond = data; if (!sysfs_kdamond->refresh_ms) return 0; - if (time_before(jiffies, damon_sysfs_next_update_jiffies)) + if (time_before(jiffies, sysfs_kdamond->next_refresh_jiffies)) return 0; - damon_sysfs_next_update_jiffies = jiffies + + sysfs_kdamond->next_refresh_jiffies = jiffies + msecs_to_jiffies(sysfs_kdamond->refresh_ms); if (!mutex_trylock(&damon_sysfs_lock)) @@ -2273,14 +2600,15 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) } kdamond->damon_ctx = ctx; - damon_sysfs_next_update_jiffies = - jiffies + msecs_to_jiffies(kdamond->refresh_ms); + kdamond->next_refresh_jiffies = jiffies + + msecs_to_jiffies(kdamond->refresh_ms); repeat_call_control->fn = damon_sysfs_repeat_call_fn; repeat_call_control->data = kdamond; repeat_call_control->repeat = true; repeat_call_control->dealloc_on_cancel = true; - if (damon_call(ctx, repeat_call_control)) + err = damon_call(ctx, repeat_call_control); + if (err) kfree(repeat_call_control); return err; } diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 4a536d41cdb2..df84d9cc7d20 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -152,6 +152,8 @@ static void damon_test_split_at(struct kunit *test) } r->nr_accesses = 42; r->last_nr_accesses = 15; + r->probe_hits[0] = 7; + r->last_probe_hits[0] = 3; r->age = 10; damon_add_region(r, t); damon_split_region_at(t, r, 25); @@ -168,6 +170,8 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses); KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); + KUNIT_EXPECT_EQ(test, r->probe_hits[0], r_new->probe_hits[0]); + KUNIT_EXPECT_EQ(test, r->last_probe_hits[0], r_new->last_probe_hits[0]); KUNIT_EXPECT_EQ(test, r->age, r_new->age); out: @@ -189,6 +193,7 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = 10; + r->probe_hits[0] = 6; r->age = 9; damon_add_region(r, t); r2 = damon_new_region(100, 300); @@ -197,6 +202,7 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "second region alloc fail"); } r2->nr_accesses = 20; + r2->probe_hits[0] = 14; r2->age = 21; damon_add_region(r2, t); @@ -204,6 +210,7 @@ static void damon_test_merge_two(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); + KUNIT_EXPECT_EQ(test, r->probe_hits[0], 11); KUNIT_EXPECT_EQ(test, r->age, 17u); i = 0; @@ -434,17 +441,17 @@ static void damon_test_ops_registration(struct kunit *test) KUNIT_EXPECT_EQ(test, damon_select_ops(c, NR_DAMON_OPS), -EINVAL); /* Registration should success after unregistration */ - mutex_lock(&damon_ops_lock); - bak = damon_registered_ops[DAMON_OPS_VADDR]; - damon_registered_ops[DAMON_OPS_VADDR] = (struct damon_operations){}; - mutex_unlock(&damon_ops_lock); + scoped_guard(mutex, &damon_ops_lock) { + bak = damon_registered_ops[DAMON_OPS_VADDR]; + damon_registered_ops[DAMON_OPS_VADDR] = + (struct damon_operations){}; + } ops.id = DAMON_OPS_VADDR; KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), 0); - mutex_lock(&damon_ops_lock); - damon_registered_ops[DAMON_OPS_VADDR] = bak; - mutex_unlock(&damon_ops_lock); + scoped_guard(mutex, &damon_ops_lock) + damon_registered_ops[DAMON_OPS_VADDR] = bak; /* Check double-registration failure again */ KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), -EINVAL); @@ -452,10 +459,9 @@ static void damon_test_ops_registration(struct kunit *test) damon_destroy_ctx(c); if (need_cleanup) { - mutex_lock(&damon_ops_lock); - damon_registered_ops[DAMON_OPS_VADDR] = - (struct damon_operations){}; - mutex_unlock(&damon_ops_lock); + scoped_guard(mutex, &damon_ops_lock) + damon_registered_ops[DAMON_OPS_VADDR] = + (struct damon_operations){}; } } @@ -463,11 +469,12 @@ static void damon_test_set_regions_for(struct kunit *test, struct damon_addr_range *old_ranges, int sz_old_ranges, struct damon_addr_range *new_ranges, int sz_new_ranges, unsigned long min_region_sz, - struct damon_addr_range *expect_ranges, int sz_expect_ranges) + struct damon_addr_range *expect_ranges, int sz_expect_ranges, + int expect_err) { struct damon_target *t; struct damon_region *r; - int i; + int i, err; t = damon_new_target(); if (!t) @@ -481,7 +488,8 @@ static void damon_test_set_regions_for(struct kunit *test, damon_add_region(r, t); } - damon_set_regions(t, new_ranges, sz_new_ranges, min_region_sz); + err = damon_set_regions(t, new_ranges, sz_new_ranges, min_region_sz); + KUNIT_EXPECT_EQ(test, err, expect_err); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), sz_expect_ranges); if (damon_nr_regions(t) != sz_expect_ranges) { @@ -510,7 +518,7 @@ static void damon_test_set_regions(struct kunit *test) (struct damon_addr_range[]){ {.start = 5, .end = 15}, {.start = 15, .end = 25}, - }, 2); + }, 2, 0); /* Un-intersecting regions should be removed. */ damon_test_set_regions_for(test, (struct damon_addr_range[]){ @@ -523,7 +531,7 @@ static void damon_test_set_regions(struct kunit *test) 1, (struct damon_addr_range[]){ {.start = 18, .end = 23}, - }, 1); + }, 1, 0); /* * Holes should be filled up with new regions. * @@ -544,7 +552,7 @@ static void damon_test_set_regions(struct kunit *test) {.start = 8, .end = 16}, {.start = 16, .end = 24}, {.start = 24, .end = 28}, - }, 3); + }, 3, 0); /* * New regions should be able to be appended. * @@ -566,7 +574,7 @@ static void damon_test_set_regions(struct kunit *test) {.start = 0, .end = 4}, {.start = 4, .end = 15}, {.start = 25, .end = 40}, - }, 3); + }, 3, 0); /* * New regions should be able to be inserted. * @@ -589,7 +597,64 @@ static void damon_test_set_regions(struct kunit *test) {.start = 0, .end = 15}, {.start = 25, .end = 40}, {.start = 44, .end = 50}, - }, 3); + }, 3, 0); + /* Zero size regions should return -EINVAL. */ + damon_test_set_regions_for(test, + (struct damon_addr_range[]){}, 0, + (struct damon_addr_range[]){ + {.start = 42, .end = 42}, + }, 1, 1, + (struct damon_addr_range[]){}, 0, -EINVAL); + /* Negative size regions should return -EINVAL. */ + damon_test_set_regions_for(test, + (struct damon_addr_range[]){}, 0, + (struct damon_addr_range[]){ + {.start = 42, .end = 21}, + }, 1, 1, + (struct damon_addr_range[]){}, 0, -EINVAL); + /* + * Regions resulting in same region after alignment should return + * -EINVAL. + */ + damon_test_set_regions_for(test, + (struct damon_addr_range[]){}, 0, + (struct damon_addr_range[]){ + {.start = 10, .end = 20}, + {.start = 20, .end = 30}, + }, 2, 4096, + (struct damon_addr_range[]){}, 0, -EINVAL); +} + +static void damon_test_nr_samples_per_aggr(struct kunit *test) +{ + struct damon_attrs attrs; + + /* Zero sample interval is treated as one. */ + attrs = (struct damon_attrs){ + .sample_interval = 0, + .aggr_interval = 5000, + }; + KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), 5000); + + /* Zero sample and aggregation intervals cover the zero-result fallback. */ + attrs = (struct damon_attrs){ + .sample_interval = 0, + .aggr_interval = 0, + }; + KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), 1); + + /* + * Too large aggregation interval on 64 bit system doesn't cause + * overflow + */ + if (ULONG_MAX > UINT_MAX) { + attrs = (struct damon_attrs){ + .sample_interval = 1, + .aggr_interval = (unsigned long)UINT_MAX + 1, + }; + KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), + UINT_MAX); + } } static void damon_test_update_monitoring_result(struct kunit *test) @@ -751,19 +816,16 @@ static void damos_test_commit_quota_goal_for(struct kunit *test, struct damos_quota_goal *dst, struct damos_quota_goal *src) { - u64 dst_last_psi_total = 0; - - if (dst->metric == DAMOS_QUOTA_SOME_MEM_PSI_US) - dst_last_psi_total = dst->last_psi_total; damos_commit_quota_goal(dst, src); KUNIT_EXPECT_EQ(test, dst->metric, src->metric); KUNIT_EXPECT_EQ(test, dst->target_value, src->target_value); if (src->metric == DAMOS_QUOTA_USER_INPUT) KUNIT_EXPECT_EQ(test, dst->current_value, src->current_value); - if (dst_last_psi_total && src->metric == DAMOS_QUOTA_SOME_MEM_PSI_US) - KUNIT_EXPECT_EQ(test, dst->last_psi_total, dst_last_psi_total); switch (dst->metric) { + case DAMOS_QUOTA_SOME_MEM_PSI_US: + KUNIT_EXPECT_EQ(test, dst->last_psi_total, U64_MAX); + break; case DAMOS_QUOTA_NODE_MEM_USED_BP: case DAMOS_QUOTA_NODE_MEM_FREE_BP: KUNIT_EXPECT_EQ(test, dst->nid, src->nid); @@ -773,6 +835,9 @@ static void damos_test_commit_quota_goal_for(struct kunit *test, KUNIT_EXPECT_EQ(test, dst->nid, src->nid); KUNIT_EXPECT_EQ(test, dst->memcg_id, src->memcg_id); break; + case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: + KUNIT_EXPECT_EQ(test, dst->nid, src->nid); + break; default: break; } @@ -787,6 +852,13 @@ static void damos_test_commit_quota_goal(struct kunit *test) .last_psi_total = 456, }; + damos_test_commit_quota_goal_for(test, &dst, + &(struct damos_quota_goal) { + .metric = DAMOS_QUOTA_SOME_MEM_PSI_US, + .target_value = 234, + .current_value = 345, + .last_psi_total = 567, + }); damos_test_commit_quota_goal_for(test, &dst, &(struct damos_quota_goal){ .metric = DAMOS_QUOTA_USER_INPUT, @@ -829,6 +901,13 @@ static void damos_test_commit_quota_goal(struct kunit *test) .current_value = 345, .last_psi_total = 567, }); + damos_test_commit_quota_goal_for(test, &dst, + &(struct damos_quota_goal){ + .metric = DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, + .target_value = 12, + .current_value = 345, + .nid = 6, + }); } static void damos_test_commit_quota_goals_for(struct kunit *test, @@ -1058,9 +1137,15 @@ static void damos_test_commit_dests(struct kunit *test) } static void damos_test_commit_filter_for(struct kunit *test, - struct damos_filter *dst, struct damos_filter *src) + struct damos_filter *dst, struct damos_filter *src, + bool expect_fail) { - damos_commit_filter(dst, src); + int err; + + err = damos_commit_filter(dst, src); + KUNIT_EXPECT_EQ(test, err != 0, expect_fail); + if (expect_fail) + return; KUNIT_EXPECT_EQ(test, dst->type, src->type); KUNIT_EXPECT_EQ(test, dst->matching, src->matching); KUNIT_EXPECT_EQ(test, dst->allow, src->allow); @@ -1081,6 +1166,10 @@ static void damos_test_commit_filter_for(struct kunit *test, KUNIT_EXPECT_EQ(test, dst->sz_range.min, src->sz_range.min); KUNIT_EXPECT_EQ(test, dst->sz_range.max, src->sz_range.max); break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + KUNIT_EXPECT_EQ(test, dst->range_min, src->range_min); + KUNIT_EXPECT_EQ(test, dst->range_max, src->range_max); + break; default: break; } @@ -1099,47 +1188,77 @@ static void damos_test_commit_filter(struct kunit *test) .type = DAMOS_FILTER_TYPE_ANON, .matching = true, .allow = true, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_MEMCG, .matching = false, .allow = false, .memcg_id = 123, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_YOUNG, .matching = true, .allow = true, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_HUGEPAGE_SIZE, .matching = false, .allow = false, .sz_range = {.min = 234, .max = 345}, - }); + }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_HUGEPAGE_SIZE, + .matching = false, + .allow = false, + .sz_range = {.min = 456, .max = 123}, + }, true); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_UNMAPPED, .matching = true, .allow = true, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_ADDR, .matching = false, .allow = false, .addr_range = {.start = 456, .end = 567}, - }); + }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_ADDR, + .matching = false, + .allow = false, + .addr_range = {.start = 567, .end = 456}, + }, true); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_TARGET, .matching = true, .allow = true, .target_idx = 6, - }); + }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, + .matching = false, + .allow = true, + .range_min = 12, + .range_max = 34, + }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, + .matching = false, + .allow = true, + .range_min = 34, + .range_max = 12, + }, true); } static void damos_test_help_initailize_scheme(struct damos *scheme) @@ -1306,6 +1425,128 @@ static void damon_test_commit_target_regions(struct kunit *test) (unsigned long[][2]) {{3, 8}, {8, 10}}, 2); } +static void damon_test_commit_filter_for(struct kunit *test, + struct damon_filter *dst, struct damon_filter *src) +{ + damon_commit_filter(dst, src); + KUNIT_EXPECT_EQ(test, dst->type, src->type); + KUNIT_EXPECT_EQ(test, dst->matching, src->matching); + KUNIT_EXPECT_EQ(test, dst->allow, src->allow); + switch (src->type) { + case DAMON_FILTER_TYPE_MEMCG: + KUNIT_EXPECT_EQ(test, dst->memcg_id, src->memcg_id); + break; + default: + break; + } +} + +static void damon_test_commit_filter(struct kunit *test) +{ + struct damon_filter dst = { + .type = DAMON_FILTER_TYPE_ANON, + .matching = false, + .allow = false, + }; + + damon_test_commit_filter_for(test, &dst, + &(struct damon_filter){ + .type = DAMON_FILTER_TYPE_ANON, + .matching = true, + .allow = true, + }); + damon_test_commit_filter_for(test, &dst, + &(struct damon_filter){ + .type = DAMON_FILTER_TYPE_MEMCG, + .matching = false, + .allow = false, + .memcg_id = 123, + }); +} + +static struct damon_ctx *damon_test_help_setup_probes(unsigned int weights[], + int nr_weights) +{ + struct damon_ctx *ctx; + struct damon_probe *probe; + int i; + + ctx = damon_new_ctx(); + if (!ctx) + return NULL; + for (i = 0; i < nr_weights; i++) { + probe = damon_new_probe(); + if (!probe) { + damon_destroy_ctx(ctx); + return NULL; + } + probe->weight = weights[i]; + damon_add_probe(ctx, probe); + } + return ctx; +} + +static void damon_test_commit_probes_for(struct kunit *test, + unsigned int dst_weights[], int nr_dst_probes, + unsigned int src_weights[], int nr_src_probes) +{ + struct damon_ctx *dst, *src; + int err; + struct damon_probe *dst_probe, *src_probe; + + dst = damon_test_help_setup_probes(dst_weights, nr_dst_probes); + if (!dst) + kunit_skip(test, "dst alloc fail"); + src = damon_test_help_setup_probes(src_weights, nr_src_probes); + if (!src) { + damon_destroy_ctx(dst); + kunit_skip(test, "src alloc fail"); + } + + err = damon_commit_probes(dst, src); + KUNIT_EXPECT_EQ(test, err, 0); + if (err) + goto out; + nr_dst_probes = 0; + damon_for_each_probe(dst_probe, dst) + nr_dst_probes++; + nr_src_probes = 0; + damon_for_each_probe(src_probe, src) + nr_src_probes++; + KUNIT_EXPECT_EQ(test, nr_dst_probes, nr_src_probes); + if (nr_dst_probes != nr_src_probes) + goto out; + nr_dst_probes = 0; + damon_for_each_probe(dst_probe, dst) { + src_probe = damon_nth_probe(nr_dst_probes, src); + KUNIT_EXPECT_EQ(test, src_probe->weight, dst_probe->weight); + nr_dst_probes++; + } +out: + damon_destroy_ctx(dst); + damon_destroy_ctx(src); +} + +static void damon_test_commit_probes(struct kunit *test) +{ + damon_test_commit_probes_for(test, + (unsigned int[]){}, 0, (unsigned int[]){}, 0); + damon_test_commit_probes_for(test, + (unsigned int[]){}, 0, (unsigned int[]){1}, 1); + damon_test_commit_probes_for(test, + (unsigned int[]){}, 0, (unsigned int[]){1, 2}, 2); + damon_test_commit_probes_for(test, + (unsigned int[]){1}, 1, (unsigned int[]){2}, 1); + damon_test_commit_probes_for(test, + (unsigned int[]){1}, 1, (unsigned int[]){2, 3}, 2); + damon_test_commit_probes_for(test, + (unsigned int[]){2, 3}, 2, (unsigned int[]){1}, 1); + damon_test_commit_probes_for(test, + (unsigned int[]){2, 3}, 2, (unsigned int[]){}, 0); + damon_test_commit_probes_for(test, + (unsigned int[]){2}, 1, (unsigned int[]){}, 0); +} + static void damon_test_commit_ctx(struct kunit *test) { struct damon_ctx *src, *dst; @@ -1331,6 +1572,62 @@ static void damon_test_commit_ctx(struct kunit *test) damon_destroy_ctx(dst); } +static void damon_test_valid_probe_params(struct kunit *test) +{ + struct damon_ctx *ctx; + struct damon_probe *probe, *probe2; + + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + probe = damon_new_probe(); + if (!probe) { + damon_destroy_ctx(ctx); + kunit_skip(test, "probe alloc fail"); + } + damon_add_probe(ctx, probe); + + /* Parameters are validated only if any probe weight is set. */ + ctx->attrs.sample_interval = 1; + ctx->attrs.aggr_interval = 1000000; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* Up to U8_MAX samples per aggregation interval are allowed. */ + probe->weight = 100; + ctx->attrs.aggr_interval = 255; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* More samples could overflow the probe_hits counters. */ + ctx->attrs.aggr_interval = 256; + KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx)); + + /* The largest weight whose weighted hit count fits in unsigned int. */ + ctx->attrs.aggr_interval = 255; + probe->weight = UINT_MAX / 255; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* Any larger weight could overflow its weighted hit count. */ + probe->weight = UINT_MAX / 255 + 1; + KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx)); + + /* With one sample per aggregation, even the largest weight fits. */ + ctx->attrs.aggr_interval = 1; + probe->weight = UINT_MAX; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* The sum of all probes' weighted hit counts could also overflow. */ + probe2 = damon_new_probe(); + if (!probe2) { + damon_destroy_ctx(ctx); + kunit_skip(test, "probe2 alloc fail"); + } + probe2->weight = 1; + damon_add_probe(ctx, probe2); + KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx)); + + damon_destroy_ctx(ctx); +} + static void damos_test_filter_out(struct kunit *test) { struct damon_target *t; @@ -1409,6 +1706,319 @@ out: damos_free_filter(f); } +static unsigned long damos_test_apply_scheme_stub(struct damon_ctx *c, + struct damon_target *t, struct damon_region *r, + struct damos *s, unsigned long *sz_filter_passed) +{ + return 0; +} + +static void damos_test_apply_scheme_filtered_sz(struct kunit *test) +{ + struct damos_access_pattern pattern = { + .min_sz_region = 0, + .max_sz_region = ULONG_MAX, + .min_nr_accesses = 0, + .max_nr_accesses = UINT_MAX, + .min_age_region = 0, + .max_age_region = UINT_MAX, + }; + unsigned long min_sz = DAMON_MIN_REGION_SZ; + struct damos_watermarks wmarks = {}; + struct damos_quota quota = {}; + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + struct damos_filter *f; + struct damos *s; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + c->ops.apply_scheme = damos_test_apply_scheme_stub; + + s = damon_new_scheme(&pattern, DAMOS_STAT, 0, "a, &wmarks, + NUMA_NO_NODE); + if (!s) { + damon_destroy_ctx(c); + kunit_skip(test, "scheme alloc fail"); + } + damon_add_scheme(c, s); + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + damon_add_target(c, t); + + f = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false); + if (!f) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f->addr_range = (struct damon_addr_range){ + .start = 2 * min_sz, + .end = 8 * min_sz + }; + damos_add_filter(s, f); + damos_set_filters_default_reject(s); + + /* reject filter, region starting before the range */ + r = damon_new_region(0, 4 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + if (damon_nr_regions(t) != 2) + goto out; + damon_destroy_region(damon_next_region(r), t); + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + + /* allow filter, region starting inside the range */ + f->allow = true; + damos_set_filters_default_reject(s); + r = damon_new_region(2 * min_sz, 10 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + KUNIT_EXPECT_EQ(test, r->ar.end, 8 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 6 * min_sz); + +out: + damon_destroy_ctx(c); +} + +static void damos_test_apply_scheme_filter_sz_unchanged(struct kunit *test) +{ + struct damos_access_pattern pattern = { + .min_sz_region = 0, + .max_sz_region = ULONG_MAX, + .min_nr_accesses = 0, + .max_nr_accesses = UINT_MAX, + .min_age_region = 0, + .max_age_region = UINT_MAX, + }; + unsigned long min_sz = DAMON_MIN_REGION_SZ; + struct damos_watermarks wmarks = {}; + struct damos_quota quota = {}; + struct damos_filter *f, *f2; + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + struct damos *s; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + c->ops.apply_scheme = damos_test_apply_scheme_stub; + + s = damon_new_scheme(&pattern, DAMOS_STAT, 0, "a, &wmarks, + NUMA_NO_NODE); + if (!s) { + damon_destroy_ctx(c); + kunit_skip(test, "scheme alloc fail"); + } + damon_add_scheme(c, s); + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + damon_add_target(c, t); + + f = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false); + if (!f) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f->addr_range = (struct damon_addr_range){ + .start = 2 * min_sz, .end = 8 * min_sz}; + damos_add_filter(s, f); + damos_set_filters_default_reject(s); + + /* wholly inside a reject range: not counted at all */ + r = damon_new_region(4 * min_sz, 6 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.nr_tried, 0); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 0); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* wholly inside an allow range: counted whole, not split */ + f->allow = true; + damos_set_filters_default_reject(s); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* two filters, each trimming: the size left after both is counted */ + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + f->allow = false; + f->addr_range = (struct damon_addr_range){ + .start = 4 * min_sz, .end = 12 * min_sz}; + f2 = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false); + if (!f2) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f2->addr_range = (struct damon_addr_range){ + .start = 2 * min_sz, .end = 3 * min_sz}; + damos_add_filter(s, f2); + damos_set_filters_default_reject(s); + + r = damon_new_region(0, 8 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + /* a core filter that never splits: counted whole */ + damos_destroy_filter(f2); + damos_destroy_filter(f); + f = damos_new_filter(DAMOS_FILTER_TYPE_TARGET, true, true); + if (!f) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f->target_idx = 0; + damos_add_filter(s, f); + damos_set_filters_default_reject(s); + s->stat = (struct damos_stat){}; + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + damon_destroy_ctx(c); +} + +static void damos_test_apply_scheme_quota_sz(struct kunit *test) +{ + struct damos_access_pattern pattern = { + .min_sz_region = 0, + .max_sz_region = ULONG_MAX, + .min_nr_accesses = 0, + .max_nr_accesses = UINT_MAX, + .min_age_region = 0, + .max_age_region = UINT_MAX, + }; + unsigned long min_sz = DAMON_MIN_REGION_SZ; + struct damos_watermarks wmarks = {}; + struct damos_quota quota = {}; + struct damon_region *r, *next; + struct damon_ctx *c; + struct damon_target *t; + struct damos *s; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + s = damon_new_scheme(&pattern, DAMOS_STAT, 0, "a, &wmarks, + NUMA_NO_NODE); + if (!s) { + damon_destroy_ctx(c); + kunit_skip(test, "scheme alloc fail"); + } + damon_add_scheme(c, s); + damos_set_filters_default_reject(s); + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + damon_add_target(c, t); + + /* no apply_scheme operation: the whole region is counted */ + r = damon_new_region(0, 4 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 4 * min_sz); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* no filter and no quota: the whole region is counted */ + c->ops.apply_scheme = damos_test_apply_scheme_stub; + s->stat = (struct damos_stat){}; + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 4 * min_sz); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* the quota trims the region: the trimmed size is counted */ + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + s->quota.esz = 2 * min_sz; + s->quota.charged_sz = 0; + + r = damon_new_region(0, 8 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + /* quota remainder below one region: tried, but nothing counted */ + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + s->quota.esz = 1; + s->quota.charged_sz = 0; + + r = damon_new_region(0, 4 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.nr_tried, 1); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 0); + KUNIT_EXPECT_EQ(test, r->ar.end, 4 * min_sz); + + damon_destroy_ctx(c); +} + static void damon_test_feed_loop_next_input(struct kunit *test) { unsigned long last_input = 900000, current_score = 200; @@ -1643,6 +2253,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_split_above_half_progresses), KUNIT_CASE(damon_test_ops_registration), KUNIT_CASE(damon_test_set_regions), + KUNIT_CASE(damon_test_nr_samples_per_aggr), KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), KUNIT_CASE(damon_test_mvsum), @@ -1656,8 +2267,14 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damos_test_commit_pageout), KUNIT_CASE(damos_test_commit_migrate_hot), KUNIT_CASE(damon_test_commit_target_regions), + KUNIT_CASE(damon_test_commit_filter), + KUNIT_CASE(damon_test_commit_probes), KUNIT_CASE(damon_test_commit_ctx), + KUNIT_CASE(damon_test_valid_probe_params), KUNIT_CASE(damos_test_filter_out), + KUNIT_CASE(damos_test_apply_scheme_filtered_sz), + KUNIT_CASE(damos_test_apply_scheme_filter_sz_unchanged), + KUNIT_CASE(damos_test_apply_scheme_quota_sz), KUNIT_CASE(damon_test_feed_loop_next_input), KUNIT_CASE(damon_test_set_filters_default_reject), KUNIT_CASE(damon_test_apply_min_nr_regions), diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 04ee2a2c6a4d..d5dde97b3cd0 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -189,22 +189,12 @@ static int damon_va_three_regions(struct damon_target *t, * * */ -static void __damon_va_init_regions(struct damon_ctx *ctx, - struct damon_target *t) +static void __damon_va_init_regions(struct damon_target *t) { - struct damon_target *ti; struct damon_addr_range regions[3]; - int tidx = 0; - if (damon_va_three_regions(t, regions)) { - damon_for_each_target(ti, ctx) { - if (ti == t) - break; - tidx++; - } - pr_debug("Failed to get three regions of %dth target\n", tidx); + if (damon_va_three_regions(t, regions)) return; - } damon_set_regions(t, regions, 3, DAMON_MIN_REGION_SZ); } @@ -217,7 +207,7 @@ static void damon_va_init(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { /* the user may set the target regions as they want */ if (!damon_nr_regions(t)) - __damon_va_init_regions(ctx, t); + __damon_va_init_regions(t); } } @@ -293,40 +283,6 @@ out: } #ifdef CONFIG_HUGETLB_PAGE -static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long addr, pte_t *entry) -{ - unsigned long psize = huge_page_size(hstate_vma(vma)); - - if (!pte_young(*entry)) - return false; - *entry = huge_ptep_get_and_clear(mm, addr, pte, psize); - *entry = pte_mkold(*entry); - set_huge_pte_at(mm, addr, pte, *entry, psize); - return true; -} - -static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long addr) -{ - bool referenced = false; - pte_t entry = huge_ptep_get(mm, addr, pte); - struct folio *folio = pfn_folio(pte_pfn(entry)); - - folio_get(folio); - - referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry); - if (mmu_notifier_clear_young(mm, addr, - addr + huge_page_size(hstate_vma(vma)))) - referenced = true; - - if (referenced) - folio_set_young(folio); - - folio_set_idle(folio); - folio_put(folio); -} - static int damon_mkold_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -527,6 +483,243 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) return max_nr_accesses; } +static void damon_va_prep_probe_region(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r, + struct damon_probe *probe) +{ + struct damon_prep *p; + + damon_for_each_prep(p, probe) { + switch (p->action) { + case DAMON_PREP_SET_PGIDLE: + damon_va_mkold(mm, r->sampling_addr); + break; + default: + break; + } + } +} + +static void damon_va_prep_probes(struct damon_ctx *ctx, bool set_samples) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + struct damon_probe *p; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + damon_for_each_region(r, t) { + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); + damon_for_each_probe(p, ctx) + damon_va_prep_probe_region(ctx, mm, r, p); + } + mmput(mm); + } +} + +static bool damon_va_young_addr(struct folio *folio, pte_t *pte, pmd_t *pmd, + struct mm_struct *mm, unsigned long addr) +{ + bool young = false; + + if (pte) + young = pte_young(*pte); + else if (pmd) + young = pmd_young(*pmd); + young = young || !folio_test_idle(folio) || + mmu_notifier_test_young(mm, addr); + return young; +} + +static bool damon_va_filter_match(struct damon_filter *filter, + struct folio *folio, pte_t *pte, pmd_t *pmd, + struct mm_struct *mm, unsigned long addr) +{ + bool matched = false; + + switch (filter->type) { + case DAMON_FILTER_TYPE_PGIDLE_UNSET: + if (!folio) + matched = false; + else + matched = damon_va_young_addr(folio, pte, pmd, mm, + addr); + break; + case DAMON_FILTER_TYPE_PGIDLE_SET: + if (!folio) + matched = false; + else + matched = !damon_va_young_addr(folio, pte, pmd, mm, + addr); + break; + default: + return damon_ops_filter_match(filter, folio); + } + return matched == filter->matching; +} + +static bool damon_va_filter_pass(struct folio *folio, struct damon_probe *p, + pte_t *pte, pmd_t *pmd, struct mm_struct *mm, + unsigned long addr) +{ + struct damon_filter *f; + bool pass = true; + + damon_for_each_filter(f, p) { + if (damon_va_filter_match(f, folio, pte, pmd, mm, addr)) { + pass = f->allow; + break; + } + pass = !f->allow; + } + return pass; +} + +struct damon_va_probe_walk_private { + struct damon_ctx *ctx; + struct damon_region *r; +}; + +static void damon_va_probe_folio(struct damon_ctx *ctx, + struct damon_region *r, struct folio *folio, + pte_t *pte, pmd_t *pmd, struct mm_struct *mm) +{ + struct damon_probe *probe; + int i = 0; + + damon_for_each_probe(probe, ctx) { + if (damon_va_filter_pass(folio, probe, pte, pmd, mm, + r->sampling_addr)) + r->probe_hits[i]++; + i++; + } +} + +static int damon_va_probe_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t *pte; + pte_t ptent; + spinlock_t *ptl; + struct folio *folio; + struct damon_va_probe_walk_private *priv = walk->private; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + ptl = pmd_trans_huge_lock(pmd, walk->vma); + if (ptl) { + pmd_t pmde = pmdp_get(pmd); + + if (!pmd_present(pmde)) + goto huge_out; + folio = vm_normal_folio_pmd(walk->vma, addr, pmde); + if (!folio) + goto huge_out; + damon_va_probe_folio(priv->ctx, priv->r, folio, NULL, &pmde, + walk->vma->vm_mm); + +huge_out: + spin_unlock(ptl); + return 0; + } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + + pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte) + return 0; + ptent = ptep_get(pte); + if (!pte_present(ptent)) + goto out; + folio = vm_normal_folio(walk->vma, addr, ptent); + if (!folio) + goto out; + damon_va_probe_folio(priv->ctx, priv->r, folio, &ptent, NULL, + walk->vma->vm_mm); + +out: + pte_unmap_unlock(pte, ptl); + return 0; +} + +#ifdef CONFIG_HUGETLB_PAGE +static int damon_va_probe_hugetlb_entry(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, struct mm_walk *walk) +{ + struct damon_va_probe_walk_private *priv = walk->private; + struct hstate *h = hstate_vma(walk->vma); + struct folio *folio; + spinlock_t *ptl; + pte_t entry; + + ptl = huge_pte_lock(h, walk->mm, pte); + entry = huge_ptep_get(walk->mm, addr, pte); + if (!pte_present(entry)) + goto out; + + folio = pfn_folio(pte_pfn(entry)); + folio_get(folio); + damon_va_probe_folio(priv->ctx, priv->r, folio, &entry, NULL, + walk->vma->vm_mm); + folio_put(folio); + +out: + spin_unlock(ptl); + return 0; +} +#else +#define damon_va_probe_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ + +static void __damon_va_apply_probes(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r) +{ + struct damon_va_probe_walk_private arg = { + .ctx = ctx, + .r = r, + }; + struct mm_walk_ops damon_probe_walk_ops = { + .pmd_entry = damon_va_probe_pmd_entry, + .hugetlb_entry = damon_va_probe_hugetlb_entry, + }; + unsigned long addr = r->sampling_addr; + + if (!mm) + return; + + damon_va_walk_page_range(mm, addr, addr + 1, &damon_probe_walk_ops, + &arg); +} + +static unsigned int damon_va_apply_probes(struct damon_ctx *ctx, + bool set_samples, bool return_max_wsum) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + unsigned int max_wsum = 0; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + damon_for_each_region(r, t) { + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); + __damon_va_apply_probes(ctx, mm, r); + if (return_max_wsum) + max_wsum = max(damon_probe_hits_wsum(r, false, + false, ctx), max_wsum); + } + if (mm) + mmput(mm); + } + + return max_wsum; +} + static bool damos_va_filter_young_match(struct damos_filter *filter, struct folio *folio, struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pmd_t *pmdp) @@ -676,6 +869,8 @@ huge_out: return 0; for (; addr < next; pte += nr, addr += nr * PAGE_SIZE) { + unsigned long page_idx; + nr = 1; ptent = ptep_get(pte); @@ -688,7 +883,8 @@ huge_out: continue; damos_va_migrate_dests_add(folio, walk->vma, addr, dests, migration_lists); - nr = folio_nr_pages(folio); + page_idx = folio_page_idx(folio, pte_page(ptent)); + nr = folio_nr_pages(folio) - page_idx; } pte_unmap_unlock(start_pte, ptl); return 0; @@ -838,6 +1034,8 @@ huge_unlock: return 0; for (; addr < next; pte += nr, addr += nr * PAGE_SIZE) { + unsigned long page_idx; + nr = 1; ptent = ptep_get(pte); @@ -851,7 +1049,8 @@ huge_unlock: if (!damos_va_filter_out(s, folio, vma, addr, pte, NULL)) *sz_filter_passed += folio_size(folio); - nr = folio_nr_pages(folio); + page_idx = folio_page_idx(folio, pte_page(ptent)); + nr = folio_nr_pages(folio) - page_idx; s->last_applied = folio; } pte_unmap_unlock(start_pte, ptl); @@ -949,6 +1148,8 @@ static int __init damon_va_initcall(void) .update = damon_va_update, .prepare_access_checks = damon_va_prepare_access_checks, .check_accesses = damon_va_check_accesses, + .prep_probes = damon_va_prep_probes, + .apply_probes = damon_va_apply_probes, .target_valid = damon_va_target_valid, .cleanup_target = damon_va_cleanup_target, .apply_scheme = damon_va_apply_scheme, diff --git a/mm/debug.c b/mm/debug.c index 9a0297b3988d..655e6bcc0e8d 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -157,17 +157,13 @@ void dump_vma(const struct vm_area_struct *vma) pr_emerg("vma %px start %px end %px mm %px\n" "prot %lx anon_vma %px vm_ops %px\n" "pgoff %lx file %px private_data %px\n" -#ifdef CONFIG_PER_VMA_LOCK "refcnt %x\n" -#endif "flags: %#lx(%pGv)\n", vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, (unsigned long)pgprot_val(vma->vm_page_prot), vma->anon_vma, vma->vm_ops, vma_start_pgoff(vma), vma->vm_file, vma->vm_private_data, -#ifdef CONFIG_PER_VMA_LOCK refcount_read(&vma->vm_refcnt), -#endif vma->vm_flags, &vma->vm_flags); } EXPORT_SYMBOL(dump_vma); diff --git a/mm/execmem.c b/mm/execmem.c index 74a178a87e75..349cadd87486 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -51,7 +51,8 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, } if (!p) { - pr_warn_ratelimited("unable to allocate memory\n"); + if (!(vm_flags & VM_REQUIRE_HUGE_VMAP)) + pr_warn_ratelimited("unable to allocate memory\n"); return NULL; } @@ -113,28 +114,6 @@ static inline unsigned long mas_range_len(struct ma_state *mas) return mas->last - mas->index + 1; } -static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid) -{ - unsigned int nr = (1 << get_vm_area_page_order(vm)); - unsigned int updated = 0; - int err = 0; - - for (int i = 0; i < vm->nr_pages; i += nr) { - err = set_direct_map_valid_noflush(vm->pages[i], nr, valid); - if (err) - goto err_restore; - updated += nr; - } - - return 0; - -err_restore: - for (int i = 0; i < updated; i += nr) - set_direct_map_valid_noflush(vm->pages[i], nr, !valid); - - return err; -} - static int execmem_force_rw(void *ptr, size_t size) { unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT; @@ -159,24 +138,30 @@ int execmem_restore_rox(void *ptr, size_t size) static void execmem_cache_clean(struct work_struct *work) { struct maple_tree *free_areas = &execmem_cache.free_areas; - struct mutex *mutex = &execmem_cache.mutex; MA_STATE(mas, free_areas, 0, ULONG_MAX); void *area; - mutex_lock(mutex); + guard(mutex)(&execmem_cache.mutex); mas_for_each(&mas, area, ULONG_MAX) { + struct vm_struct *vm = find_vm_area(area); size_t size = mas_range_len(&mas); - if (IS_ALIGNED(size, PMD_SIZE) && - IS_ALIGNED(mas.index, PMD_SIZE)) { - struct vm_struct *vm = find_vm_area(area); + if (vm && get_vm_area_size(vm) == size) { + VM_WARN_ON_ONCE(!IS_ALIGNED(mas.index, PMD_SIZE) || + !IS_ALIGNED(size, PMD_SIZE)); - execmem_set_direct_map_valid(vm, true); - mas_store_gfp(&mas, NULL, GFP_KERNEL); + /* + * Preallocate to ensure mas_store does not fail + * If there is no memory for the tree update, bail out, + * next execmem_free() might be more lucky + */ + if (mas_preallocate(&mas, NULL, GFP_KERNEL)) + break; + + mas_store_prealloc(&mas, NULL); vfree(area); } } - mutex_unlock(mutex); } static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean); @@ -242,30 +227,34 @@ static void *execmem_cache_alloc_locked(struct execmem_range *range, size_t size addr = mas_free.index; last = mas_free.last; + mas_set_range(&mas_free, addr, addr + size - 1); + if (mas_preallocate(&mas_free, NULL, GFP_KERNEL)) + return NULL; + /* insert allocated size to busy_areas at range [addr, addr + size) */ mas_set_range(&mas_busy, addr, addr + size - 1); err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL); if (err) - return NULL; + goto err_destroy_mas_free; - mas_store_gfp(&mas_free, NULL, GFP_KERNEL); + mas_store_prealloc(&mas_free, NULL); if (area_size > size) { - void *ptr = (void *)(addr + size); - /* * re-insert remaining free size to free_areas at range * [addr + size, last] + * the range matches an existing entry, so this cannot allocate */ + ptr = (void *)(addr + size); mas_set_range(&mas_free, addr + size, last); - err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL); - if (err) { - mas_store_gfp(&mas_busy, NULL, GFP_KERNEL); - return NULL; - } + mas_store_gfp(&mas_free, ptr, GFP_KERNEL); } ptr = (void *)addr; return ptr; + +err_destroy_mas_free: + mas_destroy(&mas_free); + return NULL; } static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) @@ -275,58 +264,55 @@ static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) return execmem_cache_alloc_locked(range, size); } -static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) +static void *execmem_vmalloc_rox(struct execmem_range *range, size_t size, + unsigned long vm_flags) { - unsigned long vm_flags = VM_ALLOW_HUGE_VMAP; - struct mutex *mutex = &execmem_cache.mutex; - struct vm_struct *vm; - size_t alloc_size; - int err = -ENOMEM; - void *p; - - alloc_size = round_up(size, PMD_SIZE); - p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); - if (!p) { - alloc_size = size; - p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); - } + void *p __free(vfree) = execmem_vmalloc(range, size, PAGE_KERNEL, vm_flags); + int err; if (!p) return NULL; - vm = find_vm_area(p); - if (!vm) - goto err_free_mem; - /* fill memory with instructions that will trap */ - execmem_fill_trapping_insns(p, alloc_size); - - err = set_memory_rox((unsigned long)p, vm->nr_pages); + execmem_fill_trapping_insns(p, size); + set_vm_flush_reset_perms(p); + err = set_memory_rox((unsigned long)p, size >> PAGE_SHIFT); if (err) - goto err_free_mem; + return NULL; + + return no_free_ptr(p); +} + +static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) +{ + unsigned long vm_flags = VM_REQUIRE_HUGE_VMAP; + size_t alloc_size = round_up(size, PMD_SIZE); + void *p __free(vfree) = NULL; + int err; + + p = execmem_vmalloc_rox(range, alloc_size, vm_flags); + if (!p) + return NULL; /* * New memory blocks must be allocated and added to the cache * as an atomic operation, otherwise they may be consumed * by a parallel call to the execmem_cache_alloc function. */ - mutex_lock(mutex); + guard(mutex)(&execmem_cache.mutex); err = execmem_cache_add_locked(p, alloc_size, GFP_KERNEL); if (err) - goto err_reset_direct_map; + return NULL; - p = execmem_cache_alloc_locked(range, size); + /* the chunk belongs to the cache now */ + retain_and_null_ptr(p); - mutex_unlock(mutex); + return execmem_cache_alloc_locked(range, size); +} - return p; - -err_reset_direct_map: - mutex_unlock(mutex); - execmem_set_direct_map_valid(vm, true); -err_free_mem: - vfree(p); - return NULL; +static void *execmem_alloc_rox(struct execmem_range *range, size_t size) +{ + return execmem_vmalloc_rox(range, size, 0); } static void *execmem_cache_alloc(struct execmem_range *range, size_t size) @@ -456,6 +442,11 @@ static void *execmem_cache_alloc(struct execmem_range *range, size_t size) return NULL; } +static void *execmem_alloc_rox(struct execmem_range *range, size_t size) +{ + return NULL; +} + static bool execmem_cache_free(void *ptr) { return false; @@ -465,17 +456,20 @@ static bool execmem_cache_free(void *ptr) void *execmem_alloc(enum execmem_type type, size_t size) { struct execmem_range *range = &execmem_info->ranges[type]; - bool use_cache = range->flags & EXECMEM_ROX_CACHE; + bool use_rox_cache = range->flags & EXECMEM_ROX_CACHE; unsigned long vm_flags = VM_FLUSH_RESET_PERMS; pgprot_t pgprot = range->pgprot; void *p = NULL; size = PAGE_ALIGN(size); - if (use_cache) + if (use_rox_cache) { p = execmem_cache_alloc(range, size); - else + if (!p) + p = execmem_alloc_rox(range, size); + } else { p = execmem_vmalloc(range, size, pgprot, vm_flags); + } return kasan_reset_tag(p); } diff --git a/mm/filemap.c b/mm/filemap.c index 00fd89cf6f55..b74bc1e5015c 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -918,14 +918,6 @@ noinline int __filemap_add_folio(struct address_space *mapping, mapping->nrpages += nr; - /* hugetlb pages do not participate in page cache accounting */ - if (!huge) { - lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); - if (folio_test_pmd_mappable(folio)) - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, nr); - } - unlock: xas_unlock_irq(&xas); @@ -942,6 +934,13 @@ unlock: if (xas_error(&xas)) goto error; + /* hugetlb pages do not participate in page cache accounting */ + if (!huge) { + lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); + if (folio_test_pmd_mappable(folio)) + lruvec_stat_mod_folio(folio, NR_FILE_THPS, nr); + } + trace_mm_filemap_add_to_page_cache(folio); return 0; error: @@ -1686,6 +1685,8 @@ EXPORT_SYMBOL_GPL(folio_end_writeback_no_dropbehind); */ void folio_end_writeback(struct folio *folio) { + bool swap_dropbehind; + VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); /* @@ -1695,7 +1696,24 @@ void folio_end_writeback(struct folio *folio) * reused before the folio_wake_bit(). */ folio_get(folio); + + /* + * Sample this before folio_end_writeback_no_dropbehind() clears + * PG_writeback: until then a racing swapin cannot remove the folio from + * the swap cache. Afterwards it can, and the drop below then finds a + * non-swapcache folio and puts it back on the LRU instead. The + * reference taken above keeps the folio alive across that window. + */ + swap_dropbehind = folio_test_swapcache(folio) && + folio_test_dropbehind(folio); + folio_end_writeback_no_dropbehind(folio); + + if (swap_dropbehind) { + swap_writeback_dropbehind_folio(folio); + return; + } + folio_end_dropbehind(folio); folio_put(folio); } diff --git a/mm/folio.c b/mm/folio.c index 50a6dbe55998..47a437e0f7fd 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -354,26 +354,28 @@ static void __lru_cache_activate_folio(struct folio *folio) static void lru_gen_inc_refs(struct folio *folio) { - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + int refs; if (folio_test_unevictable(folio)) return; /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio)) { - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); + if (!folio_lru_refs(folio)) { + folio_set_lru_refs(folio, 1); return; } do { - if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) { + new_flags = old_flags; + refs = lru_get_refs_flags(old_flags); + if (refs == LRU_REFS_MAX) { if (!folio_test_workingset(folio)) folio_set_workingset(folio); return; } - - new_flags = old_flags + BIT(LRU_REFS_PGOFF); - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); + lru_set_refs_flags(&new_flags, refs + 1); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } static bool lru_gen_clear_refs(struct folio *folio) @@ -385,7 +387,8 @@ static bool lru_gen_clear_refs(struct folio *folio) if (gen < 0) return true; - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0); + folio_set_lru_refs(folio, 0); + folio_clear_workingset(folio); rcu_read_lock(); seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]); diff --git a/mm/gup.c b/mm/gup.c index eb898ea1ee22..c2dfcb4744bc 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2700,8 +2700,9 @@ EXPORT_SYMBOL(get_user_pages_unlocked); * Before activating this code, please be aware that the following assumptions * are currently made: * - * *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to - * free pages containing page tables or TLB flushing requires IPI broadcast. + * *) tlb_remove_table() is used to free pages containing page tables, with + * the free deferred until an RCU grace period has elapsed (see + * mm/mmu_gather.c). * * *) ptes can be read atomically by the architecture. * @@ -2826,11 +2827,11 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) * also check pmd here to make sure pmd doesn't change (corresponds to * pmdp_collapse_flush() in the THP collapse code path). */ -static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - int ret = 0; + unsigned long nr_pages = 0; pte_t *ptep, *ptem; ptem = ptep = pte_offset_map(&pmd, addr); @@ -2892,15 +2893,13 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, goto pte_unmap; } folio_set_referenced(folio); - pages[*nr] = page; - (*nr)++; + pages[nr_pages] = page; + nr_pages++; } while (ptep++, addr += PAGE_SIZE, addr != end); - ret = 1; - pte_unmap: pte_unmap(ptem); - return ret; + return nr_pages; } #else @@ -2913,21 +2912,25 @@ pte_unmap: * get_user_pages_fast_only implementation that can pin pages. Thus it's still * useful to have gup_fast_pmd_leaf even if we can't operate on ptes. */ -static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { return 0; } #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ -static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { struct page *page; struct folio *folio; - int refs; + unsigned long nr_pages, i; + + /* See gup_fast_pte_range() */ + if (pmd_protnone(orig)) + return 0; if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) return 0; @@ -2935,42 +2938,40 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, if (pmd_special(orig)) return 0; - refs = (end - addr) >> PAGE_SHIFT; + nr_pages = (end - addr) >> PAGE_SHIFT; page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - folio = try_grab_folio_fast(page, refs, flags); + folio = try_grab_folio_fast(page, nr_pages, flags); if (!folio) return 0; if (unlikely(pmd_val(orig) != pmd_val(pmdp_get_lockless(pmdp)))) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!gup_fast_folio_allowed(folio, flags)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } - pages += *nr; - *nr += refs; - for (; refs; refs--) + for (i = 0; i < nr_pages; i++) *(pages++) = page++; folio_set_referenced(folio); - return 1; + return nr_pages; } -static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { struct page *page; struct folio *folio; - int refs; + unsigned long nr_pages, i; if (!pud_access_permitted(orig, flags & FOLL_WRITE)) return 0; @@ -2978,41 +2979,39 @@ static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, if (pud_special(orig)) return 0; - refs = (end - addr) >> PAGE_SHIFT; + nr_pages = (end - addr) >> PAGE_SHIFT; page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - folio = try_grab_folio_fast(page, refs, flags); + folio = try_grab_folio_fast(page, nr_pages, flags); if (!folio) return 0; if (unlikely(pud_val(orig) != pud_val(pudp_get(pudp)))) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!gup_fast_folio_allowed(folio, flags)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } - pages += *nr; - *nr += refs; - for (; refs; refs--) + for (i = 0; i < nr_pages; i++) *(pages++) = page++; folio_set_referenced(folio); - return 1; + return nr_pages; } -static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pmd_range(pud_t *pudp, pud_t pud, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pmd_t *pmdp; pmdp = pmd_offset_lockless(pudp, pud, addr); @@ -3021,30 +3020,30 @@ static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, next = pmd_addr_end(addr, end); if (!pmd_present(pmd)) - return 0; + break; if (unlikely(pmd_leaf(pmd))) { - /* See gup_fast_pte_range() */ - if (pmd_protnone(pmd)) - return 0; + chunk_nr_pages = gup_fast_pmd_leaf(pmd, pmdp, addr, + next, flags, + &pages[nr_pages]); - if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags, - pages, nr)) - return 0; - - } else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags, - pages, nr)) - return 0; + } else + chunk_nr_pages = gup_fast_pte_range(pmd, pmdp, addr, + next, flags, + &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pmdp++, addr = next, addr != end); - return 1; + return nr_pages; } -static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pud_t *pudp; pudp = pud_offset_lockless(p4dp, p4d, addr); @@ -3053,24 +3052,27 @@ static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, next = pud_addr_end(addr, end); if (unlikely(!pud_present(pud))) - return 0; - if (unlikely(pud_leaf(pud))) { - if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags, - pages, nr)) - return 0; - } else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags, - pages, nr)) - return 0; + break; + if (unlikely(pud_leaf(pud))) + chunk_nr_pages = gup_fast_pud_leaf(pud, pudp, addr, + next, flags, + &pages[nr_pages]); + else + chunk_nr_pages = gup_fast_pmd_range(pudp, pud, addr, + next, flags, + &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pudp++, addr = next, addr != end); - return 1; + return nr_pages; } -static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; p4d_t *p4dp; p4dp = p4d_offset_lockless(pgdp, pgd, addr); @@ -3079,20 +3081,23 @@ static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, next = p4d_addr_end(addr, end); if (!p4d_present(p4d)) - return 0; + break; BUILD_BUG_ON(p4d_leaf(p4d)); - if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags, - pages, nr)) - return 0; + chunk_nr_pages = gup_fast_pud_range(p4dp, p4d, addr, next, + flags, &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (p4dp++, addr = next, addr != end); - return 1; + return nr_pages; } -static void gup_fast_pgd_range(unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static unsigned long gup_fast_pgd_range(unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pgd_t *pgdp; pgdp = pgd_offset(current->mm, addr); @@ -3101,17 +3106,23 @@ static void gup_fast_pgd_range(unsigned long addr, unsigned long end, next = pgd_addr_end(addr, end); if (pgd_none(pgd)) - return; + break; BUILD_BUG_ON(pgd_leaf(pgd)); - if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags, - pages, nr)) - return; + chunk_nr_pages = gup_fast_p4d_range(pgdp, pgd, addr, next, + flags, &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pgdp++, addr = next, addr != end); + + return nr_pages; } #else -static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static inline unsigned long gup_fast_pgd_range(unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages) { + return 0; } #endif /* CONFIG_HAVE_GUP_FAST */ @@ -3129,8 +3140,7 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end) static unsigned long gup_fast(unsigned long start, unsigned long end, unsigned int gup_flags, struct page **pages) { - unsigned long flags; - int nr_pinned = 0; + unsigned long flags, nr_pinned; unsigned seq; if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || @@ -3154,7 +3164,7 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, * that come from callers of tlb_remove_table_sync_one(). */ local_irq_save(flags); - gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned); + nr_pinned = gup_fast_pgd_range(start, end, gup_flags, pages); local_irq_restore(flags); /* diff --git a/mm/gup_test.c b/mm/gup_test.c index 44c1cdfb9c37..ba74bf3f4104 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -116,7 +116,8 @@ static int __gup_test_ioctl(unsigned int cmd, bool needs_mmap_lock = cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; - if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX) + if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX || !gup->size || + !gup->nr_pages_per_call || !PAGE_ALIGNED(gup->size)) return -EINVAL; if (check_add_overflow((unsigned long)gup->addr, (unsigned long)gup->size, &end)) @@ -139,11 +140,9 @@ static int __gup_test_ioctl(unsigned int cmd, if (nr != gup->nr_pages_per_call) break; + nr = min_t(unsigned long, nr, (end - addr) / PAGE_SIZE); + next = addr + nr * PAGE_SIZE; - if (next > end) { - next = end; - nr = (next - addr) / PAGE_SIZE; - } switch (cmd) { case GUP_FAST_BENCHMARK: @@ -188,7 +187,7 @@ static int __gup_test_ioctl(unsigned int cmd, nr_pages = i; gup->get_delta_usec = ktime_us_delta(end_time, start_time); - gup->size = addr - gup->addr; + gup->size = nr_pages * PAGE_SIZE; /* * Take an un-benchmark-timed moment to verify DMA pinned diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1e5d68acf62a..d7901b8171b2 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1146,6 +1146,7 @@ subsys_initcall(hugepage_init); static int __init setup_transparent_hugepage(char *str) { int ret = 0; + if (!str) goto out; if (!strcmp(str, "always")) { @@ -1552,6 +1553,7 @@ static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, struct folio *zero_folio) { pmd_t entry; + entry = folio_mk_pmd(zero_folio, vma->vm_page_prot); entry = pmd_mkspecial(entry); pgtable_trans_huge_deposit(mm, pmd, pgtable); @@ -1725,6 +1727,9 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, (VM_PFNMAP|VM_MIXEDMAP)); BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); + if (unlikely(is_huge_zero_pfn(pfn))) + return VM_FAULT_SIGBUS; + pfnmap_setup_cachemode_pfn(pfn, &pgprot); return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write); @@ -2423,6 +2428,10 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, } folio = pmd_folio(orig_pmd); + + if (folio_is_zone_device(folio)) + goto out; + /* * If other processes are mapping this folio, we couldn't discard * the folio unless they all do MADV_FREE so let's skip the folio. @@ -2514,25 +2523,13 @@ static struct folio *normal_or_softleaf_folio_pmd(struct vm_area_struct *vma, return pmd_to_softleaf_folio(pmdval); } -static bool has_deposited_pgtable(struct vm_area_struct *vma, pmd_t pmdval, - struct folio *folio) +static bool vma_has_deposited_pgtable(struct vm_area_struct *vma) { - /* Some architectures require unconditional depositing. */ - if (arch_needs_pgtable_deposit()) - return true; - /* - * Huge zero always deposited except for DAX which handles itself, see - * set_huge_zero_folio(). + * PMDs in anonymous VMAs always have a deposited page table. PMDs in + * other VMAs only have one when required by the architecture. */ - if (is_huge_zero_pmd(pmdval)) - return !vma_is_dax(vma); - - /* - * Otherwise, only anonymous folios are deposited, see - * __do_huge_pmd_anonymous_page(). - */ - return folio && folio_test_anon(folio); + return arch_needs_pgtable_deposit() || vma_is_anonymous(vma); } /** @@ -2572,7 +2569,7 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, is_present = pmd_present(orig_pmd); folio = normal_or_softleaf_folio_pmd(vma, addr, orig_pmd, is_present); - has_deposit = has_deposited_pgtable(vma, orig_pmd, folio); + has_deposit = vma_has_deposited_pgtable(vma); if (folio) zap_huge_pmd_folio(mm, vma, orig_pmd, folio, is_present); if (has_deposit) @@ -2657,6 +2654,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { pgtable_t pgtable; + pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); pgtable_trans_huge_deposit(mm, new_pmd, pgtable); } @@ -3531,20 +3529,28 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, split_huge_pmd_if_needed(next, end); } -static void unmap_folio(struct folio *folio) +/* + * A return value of 0 does not mean that unmapping succeeded. It might + * still have failed, but remap_anon_folio() must be called afterwards, + * for anon folios. + */ +static int unmap_folio(struct folio *folio) { enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | TTU_BATCH_FLUSH; VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); + /* Racy check if we can split the page, before we split PMDs */ + if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) + return -EAGAIN; + if (folio_test_pmd_mappable(folio)) ttu_flags |= TTU_SPLIT_HUGE_PMD; /* * Anon pages need migration entries to preserve them, but file * pages can simply be left unmapped, then faulted back on demand. - * If that is ever changed (perhaps for mlock), update remap_page(). */ if (folio_test_anon(folio)) try_to_migrate(folio, ttu_flags); @@ -3552,6 +3558,8 @@ static void unmap_folio(struct folio *folio) try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); try_to_unmap_flush(); + + return 0; } static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, @@ -3629,13 +3637,17 @@ bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); } -static void remap_page(struct folio *folio, unsigned long nr, int flags) +static void remap_anon_folio(struct folio *folio, unsigned long nr, int flags) { int i = 0; - /* If unmap_folio() uses try_to_migrate() on file, remove this check */ - if (!folio_test_anon(folio)) - return; + /* + * unmap_folio() installs migration entries only for anon folios, + * so currently only anon folios need to be remapped. File folios + * stay unmapped after the split and are faulted back on demand. + */ + VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); + for (;;) { remove_migration_ptes(folio, folio, TTU_RMAP_LOCKED | flags); i += folio_nr_pages(folio); @@ -3719,7 +3731,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order, * * Note that for mapped sub-pages of an anonymous THP, * PG_anon_exclusive has been cleared in unmap_folio() and is stored in - * the migration entry instead from where remap_page() will restore it. + * the migration entry instead from where remap_anon_folio() will restore it. * We can still have PG_anon_exclusive set on effectively unmapped and * unreferenced sub-pages of an anonymous THP: we can simply drop * PG_anon_exclusive (-> PG_mappedtodisk) for these here. @@ -3755,6 +3767,10 @@ static void __split_folio_to_order(struct folio *folio, int old_order, */ VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head); + /* + * Not all folio fields are valid during a split, so open-code + * the swap entry rather than using folio_swap_entry(). + */ if (folio_test_swapcache(folio)) new_folio->swap.val = folio->swap.val + i; @@ -3799,8 +3815,8 @@ static void __split_folio_to_order(struct folio *folio, int old_order, } /** - * __split_unmapped_folio() - splits an unmapped @folio to lower order folios in - * two ways: uniform split or non-uniform split. + * __split_frozen_folio() - splits a frozen @folio to lower order folios + * in two ways: uniform split or non-uniform split. * @folio: the to-be-split folio * @new_order: the smallest order of the after split folios (since buddy * allocator like split generates folios with orders from @folio's @@ -3808,7 +3824,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order, * @split_at: in buddy allocator like split, the folio containing @split_at * will be split until its order becomes @new_order. * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller - * @mapping: @folio->mapping * @split_type: if the split is uniform or not (buddy allocator like split) * * @@ -3839,9 +3854,9 @@ static void __split_folio_to_order(struct folio *folio, int old_order, * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be * split but not to @new_order, the caller needs to check) */ -static int __split_unmapped_folio(struct folio *folio, int new_order, +static int __split_frozen_folio(struct folio *folio, int new_order, struct page *split_at, struct xa_state *xas, - struct address_space *mapping, enum split_type split_type) + enum split_type split_type) { const bool is_anon = folio_test_anon(folio); int old_order = folio_order(folio); @@ -3849,6 +3864,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, struct folio *old_folio = folio; int split_order; + /* Frozen implies unmapped, callers unmap before splitting. */ + VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio); + /* * split to new_order one order at a time. For uniform split, * folio is split to new_order directly. @@ -3862,7 +3880,7 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, if (is_anon && split_order == 1) continue; - if (mapping) { + if (xas) { /* * uniform split has xas_split_alloc() called before * irq is disabled to allocate enough memory, whereas @@ -3922,6 +3940,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, int folio_check_splittable(struct folio *folio, unsigned int new_order, enum split_type split_type) { + const bool is_anon = folio_test_anon(folio); + const bool is_swapcache = folio_test_swapcache(folio); + VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); /* * Folios that just got truncated cannot get split. Signal to the @@ -3930,11 +3951,11 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, * TODO: this will also currently refuse folios without a mapping in the * swapcache (shmem or to-be-anon folios). */ - if (!folio->mapping && !folio_test_anon(folio)) + if (!folio->mapping && !is_anon) return -EBUSY; /* order-1 is not supported for anonymous THP. */ - if (folio_test_anon(folio) && new_order == 1) + if (is_anon && new_order == 1) return -EINVAL; /* @@ -3945,9 +3966,8 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, * swapcache folio split. Only uniform split to order-0 can be used * here. */ - if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && folio_test_swapcache(folio)) { + if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && is_swapcache) return -EINVAL; - } if (is_huge_zero_folio(folio)) return -EINVAL; @@ -3955,169 +3975,310 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, if (folio_test_writeback(folio)) return -EBUSY; + /* + * A non-anon swapcache folio that still has a mapping can only be a + * shmem folio under SWAP IO, it's removed from either swap cache or + * shmem mapping afterward. There is little benefit in splitting them + * hence reject it here up front before touching anything. + */ + if (!is_anon && is_swapcache && folio->mapping) + return -EBUSY; + return 0; } -/* Number of folio references from the pagecache or the swapcache. */ -static unsigned int folio_cache_ref_count(const struct folio *folio) +/* Number of folio references from the swapcache. */ +static unsigned int folio_swapcache_ref_count(const struct folio *folio) { - if (folio_test_anon(folio) && !folio_test_swapcache(folio)) + if (!folio_test_swapcache(folio)) return 0; return folio_nr_pages(folio); } -static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order, - struct page *split_at, struct xa_state *xas, - struct address_space *mapping, bool do_lru, - struct list_head *list, enum split_type split_type, - pgoff_t end, int *nr_shmem_dropped) +static void folio_reset_partially_mapped(struct folio *folio) +{ + /* Folio must be frozen. */ + VM_WARN_ON_FOLIO(folio_ref_count(folio), folio); + + if (!folio_test_partially_mapped(folio)) + return; + + /* + * Order-1 folios have no _deferred_list. The flag is only ever set + * on folios that do, so the list can be checked after the flag. + */ + VM_WARN_ON_FOLIO(!list_empty(&folio->_deferred_list), folio); + + folio_clear_partially_mapped(folio); + mod_mthp_stat(folio_order(folio), + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); +} + +static int __folio_freeze_split_anon(struct folio *folio, + unsigned int new_order, struct page *split_at, bool do_lru, + struct list_head *list, enum split_type split_type) { struct folio *end_folio = folio_next(folio); + struct swap_cluster_info *ci = NULL; + const int old_order = folio_order(folio); struct folio *new_folio, *next; - int old_order = folio_order(folio); - struct list_lru_one *lru; - bool dequeue_deferred; + struct anon_vma *anon_vma = NULL; + enum ttu_flags ttu_flags = 0; + struct lruvec *lruvec; int ret = 0; - VM_WARN_ON_ONCE(!mapping && end); /* - * If this folio can be on the deferred split queue, lock out - * the shrinker before freezing the ref. If the shrinker sees - * a 0-ref folio, it assumes it beat folio_put() to the list - * lock and must clean up the LRU state - the same dequeue we - * will do below as part of the split. + * Unmap/remap needs the anon_vma, so we first take a reference on + * it to prevent it from disappearing, and lock it for write here, + * letting unmap_folio() walk the rmap with TTU_RMAP_LOCKED. + * + * folio_mapped() is not stable here, but it can only change in + * one direction while the folio is locked. The mapcount can drop + * to zero at any time, zap_pte_range() takes no folio lock. It + * cannot go up: swapin, migration and uffd move all lock the folio + * before mapping it, and fork only copies PTEs that already exist. + * + * So if we see the folio mapped, the worst case is an empty rmap + * walk. If we see it unmapped, it stays unmapped and needs neither + * the reference nor the lock. Anything else needs a reference + * first and folio_ref_freeze() below catches it. + * + * Note that entirely swapped-out THPs are unmapped but can be split. */ - dequeue_deferred = folio_test_anon(folio) && old_order > 1; - if (dequeue_deferred) { - struct mem_cgroup *memcg; - - rcu_read_lock(); - memcg = folio_memcg(folio); - lru = list_lru_lock(&deferred_split_lru, - folio_nid(folio), &memcg); + if (folio_mapped(folio)) { + anon_vma = folio_get_anon_vma(folio); + if (!anon_vma) + return -EBUSY; + anon_vma_lock_write(anon_vma); + ret = unmap_folio(folio); + if (ret) + goto out_unlock; } - if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { - struct swap_cluster_info *ci = NULL; - struct lruvec *lruvec; - if (dequeue_deferred) { - __list_lru_del(&deferred_split_lru, lru, - &folio->_deferred_list, folio_nid(folio)); - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(old_order, - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } - list_lru_unlock(lru); - rcu_read_unlock(); - } + local_irq_disable(); - if (mapping) { - int nr = folio_nr_pages(folio); + if (!folio_ref_freeze(folio, folio_swapcache_ref_count(folio) + 1)) { + ret = -EAGAIN; + goto out_no_split; + } - if (folio_test_pmd_mappable(folio) && - new_order < HPAGE_PMD_ORDER) { - if (folio_test_swapbacked(folio)) { - lruvec_stat_mod_folio(folio, - NR_SHMEM_THPS, -nr); - } else { - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, -nr); - } - } - } + /* Take off the deferred split queue while frozen and memcg set */ + folio_unqueue_deferred_split(folio); - if (folio_test_swapcache(folio)) { - if (mapping) { - VM_WARN_ON_ONCE_FOLIO(mapping, folio); - return -EINVAL; - } + /* + * deferred_split_scan() takes the folio off the queue before it + * splits it, so the unqueue above finds an empty list and + * leaves PG_partially_mapped set. + * Clear it here: the flag does not survive the split. + */ + folio_reset_partially_mapped(folio); - ci = swap_cluster_get_and_lock(folio); - } + if (folio_test_swapcache(folio)) + ci = swap_cluster_get_and_lock(folio); - /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ + if (do_lru) + lruvec = folio_lruvec_lock(folio); + + ret = __split_frozen_folio(folio, new_order, split_at, NULL, split_type); + + /* + * Unfreeze the after-split folios and put them back to the right + * place. Keep the head @folio frozen until the end: sub entries + * in swap cache must be updated first, so a concurrent + * swap_cache_get_folio() cannot return the head folio for a sub + * entry (folio_try_get() will fail on the head @folio until unfreeze). + */ + for (new_folio = folio_next(folio); new_folio != end_folio; + new_folio = next) { + next = folio_next(new_folio); + zone_device_private_split_cb(folio, new_folio); + folio_ref_unfreeze(new_folio, + folio_swapcache_ref_count(new_folio) + 1); if (do_lru) - lruvec = folio_lruvec_lock(folio); - - ret = __split_unmapped_folio(folio, new_order, split_at, xas, - mapping, split_type); - - /* - * Unfreeze after-split folios and put them back to the right - * list. @folio should be kept frozon until page cache - * entries are updated with all the other after-split folios - * to prevent others seeing stale page cache entries. - * As a result, new_folio starts from the next folio of - * @folio. - */ - for (new_folio = folio_next(folio); new_folio != end_folio; - new_folio = next) { - unsigned long nr_pages = folio_nr_pages(new_folio); - - next = folio_next(new_folio); - - zone_device_private_split_cb(folio, new_folio); - - folio_ref_unfreeze(new_folio, - folio_cache_ref_count(new_folio) + 1); - - if (do_lru) - lru_add_split_folio(folio, new_folio, lruvec, list); - - /* - * Anonymous folio with swap cache. - * NOTE: shmem in swap cache is not supported yet. - */ - if (ci) { - __swap_cache_replace_folio(ci, folio, new_folio); - continue; - } - - /* Anonymous folio without swap cache */ - if (!mapping) - continue; - - /* Add the new folio to the page cache. */ - if (new_folio->index < end) { - __xa_store(&mapping->i_pages, new_folio->index, - new_folio, 0); - continue; - } - - VM_WARN_ON_ONCE(!nr_shmem_dropped); - /* Drop folio beyond EOF: ->index >= end */ - if (shmem_mapping(mapping) && nr_shmem_dropped) - *nr_shmem_dropped += nr_pages; - else if (folio_test_clear_dirty(new_folio)) - folio_account_cleaned( - new_folio, inode_to_wb(mapping->host)); - __filemap_remove_folio(new_folio, NULL); - folio_put_refs(new_folio, nr_pages); - } - - zone_device_private_split_cb(folio, NULL); - /* - * Unfreeze @folio only after all page cache entries, which - * used to point to it, have been updated with new folios. - * Otherwise, a parallel folio_try_get() can grab @folio - * and its caller can see stale page cache entries. - */ - folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); - - if (do_lru) - lruvec_unlock(lruvec); - + lru_add_split_folio(folio, new_folio, lruvec, list); if (ci) - swap_cluster_unlock(ci); - } else { - if (dequeue_deferred) { - list_lru_unlock(lru); - rcu_read_unlock(); - } - return -EAGAIN; + __swap_cache_replace_folio(ci, folio, new_folio); } + zone_device_private_split_cb(folio, NULL); + folio_ref_unfreeze(folio, folio_swapcache_ref_count(folio) + 1); + + if (do_lru) + lruvec_unlock(lruvec); + if (ci) + swap_cluster_unlock(ci); +out_no_split: + local_irq_enable(); + if (anon_vma) { + if (!ret && !folio_is_device_private(folio)) + ttu_flags = TTU_USE_SHARED_ZEROPAGE; + remap_anon_folio(folio, 1 << old_order, ttu_flags); + } +out_unlock: + if (anon_vma) { + anon_vma_unlock_write(anon_vma); + put_anon_vma(anon_vma); + } + + return ret; +} + +static int __folio_freeze_split_file(struct folio *folio, + unsigned int new_order, struct page *split_at, + struct list_head *list, enum split_type split_type) +{ + const long old_nr_pages = folio_nr_pages(folio); + struct address_space *mapping = folio->mapping; + XA_STATE(xas, &mapping->i_pages, folio->index); + struct folio *end_folio = folio_next(folio); + struct mem_cgroup *memcg, *old_memcg; + struct folio *new_folio, *next; + int nr_shmem_dropped = 0; + unsigned int min_order; + struct lruvec *lruvec; + pgoff_t end; + gfp_t gfp; + int ret = 0; + + min_order = mapping_min_folio_order(mapping); + if (new_order < min_order) + return -EINVAL; + + /* + * Switch to folio's memcg as xarray node allocation can happen and + * needs to charge to it. + */ + memcg = get_mem_cgroup_from_folio(folio); + old_memcg = set_active_memcg(memcg); + + gfp = current_gfp_context(mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK); + if (!filemap_release_folio(folio, gfp)) { + ret = -EBUSY; + goto fail_free; + } + + mapping_set_update(&xas, mapping); + + if (split_type == SPLIT_TYPE_UNIFORM) { + const int old_order = folio_order(folio); + + xas_set_order(&xas, folio->index, new_order); + xas_split_alloc(&xas, folio, old_order, gfp); + if (xas_error(&xas)) { + ret = xas_error(&xas); + goto fail_free; + } + } + + i_mmap_lock_read(mapping); + + /* Currently device private folios can only back anonymous memory. */ + VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio); + + /* + * The loop below may need to trim off pages beyond + * EOF: but on 32-bit, i_size_read() takes an irq-unsafe + * seqlock, which cannot be nested inside the page tree lock. + * So note end now: i_size itself may be changed at any moment, + * but folio lock is good enough to serialize the trimming. + */ + end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); + if (shmem_mapping(mapping)) + end = shmem_fallocend(mapping->host, end); + + ret = unmap_folio(folio); + if (ret) + goto fail_mmap_unlock; + + xas_lock_irq(&xas); + + /* + * Check if the folio is present in page cache. + * We assume all tail are present too, if folio is there. + */ + if (xas_load(&xas) != folio) { + ret = -EAGAIN; + goto fail; + } + + if (!folio_ref_freeze(folio, old_nr_pages + 1)) { + ret = -EAGAIN; + goto fail; + } + + if (folio_test_pmd_mappable(folio) && new_order < HPAGE_PMD_ORDER) { + if (folio_test_swapbacked(folio)) + lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -old_nr_pages); + else + lruvec_stat_mod_folio(folio, NR_FILE_THPS, -old_nr_pages); + } + + /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ + lruvec = folio_lruvec_lock(folio); + ret = __split_frozen_folio(folio, new_order, split_at, &xas, split_type); + + /* + * Unfreeze after-split folios and put them back to the right + * list. @folio should be kept frozen until page cache + * entries are updated with all the other after-split folios + * to prevent others seeing stale page cache entries. + * As a result, new_folio starts from the next folio of + * @folio. + */ + for (new_folio = folio_next(folio); new_folio != end_folio; + new_folio = next) { + unsigned long nr_pages = folio_nr_pages(new_folio); + + /* compute next before the folio can be freed below */ + next = folio_next(new_folio); + + folio_ref_unfreeze(new_folio, + folio_nr_pages(new_folio) + 1); + + lru_add_split_folio(folio, new_folio, lruvec, list); + + /* Add the new folio to the page cache. */ + if (new_folio->index < end) { + __xa_store(&mapping->i_pages, new_folio->index, + new_folio, 0); + continue; + } + + /* Drop folio beyond EOF: ->index >= end */ + if (shmem_mapping(mapping)) + nr_shmem_dropped += nr_pages; + else if (folio_test_clear_dirty(new_folio)) + folio_account_cleaned( + new_folio, inode_to_wb(mapping->host)); + __filemap_remove_folio(new_folio, NULL); + folio_put_refs(new_folio, nr_pages); + } + + /* + * Unfreeze @folio only after all page cache entries, which + * used to point to it, have been updated with new folios. + * Otherwise, a parallel folio_try_get() can grab @folio + * and its caller can see stale page cache entries. + */ + folio_ref_unfreeze(folio, folio_nr_pages(folio) + 1); + lruvec_unlock(lruvec); +fail: + xas_unlock_irq(&xas); +fail_mmap_unlock: + if (nr_shmem_dropped) + shmem_uncharge(mapping->host, nr_shmem_dropped); + /* + * Drop the mapping while the inode is still pinned. @folio stays + * locked and present in the page cache, so eviction cannot free + * the inode yet, nothing past this point may touch the inode or + * the mapping. + */ + i_mmap_unlock_read(mapping); +fail_free: + /* Restore the previously active memcg */ + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); + xas_destroy(&xas); return ret; } @@ -4130,9 +4291,9 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n * @list: after-split folios will be put on it if non NULL * @split_type: perform uniform split or not (non-uniform split) * - * It calls __split_unmapped_folio() to perform uniform and non-uniform split. + * It calls __split_frozen_folio() to perform uniform and non-uniform split. * It is in charge of checking whether the split is supported or not and - * preparing @folio for __split_unmapped_folio(). + * preparing @folio for __split_frozen_folio(). * * After splitting, the after-split folio containing @lock_at remains locked * and others are unlocked: @@ -4146,17 +4307,11 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct page *split_at, struct page *lock_at, struct list_head *list, enum split_type split_type) { - XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); - bool is_anon = folio_test_anon(folio); - struct mem_cgroup *memcg, *old_memcg; - struct address_space *mapping = NULL; - struct anon_vma *anon_vma = NULL; + const bool is_anon = folio_test_anon(folio); + const bool is_swapcache = folio_test_swapcache(folio); int old_order = folio_order(folio); struct folio *new_folio, *next; - int nr_shmem_dropped = 0; - enum ttu_flags ttu_flags = 0; - pgoff_t end = 0; int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); @@ -4164,141 +4319,26 @@ static int __folio_split(struct folio *folio, unsigned int new_order, if (folio != page_folio(split_at) || folio != page_folio(lock_at)) { ret = -EINVAL; - goto out_no_memcg; + goto out; } if (new_order >= old_order) { ret = -EINVAL; - goto out_no_memcg; + goto out; } ret = folio_check_splittable(folio, new_order, split_type); if (ret) { VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio"); - goto out_no_memcg; + goto out; } - /* - * switch to folio's memcg as xarray node allocation can happen and - * needs to charge to it. - */ - memcg = get_mem_cgroup_from_folio(folio); - old_memcg = set_active_memcg(memcg); - - if (is_anon) { - /* - * The caller does not necessarily hold an mmap_lock that would - * prevent the anon_vma disappearing so we first we take a - * reference to it and then lock the anon_vma for write. This - * is similar to folio_lock_anon_vma_read except the write lock - * is taken to serialise against parallel split or collapse - * operations. - */ - anon_vma = folio_get_anon_vma(folio); - if (!anon_vma) { - ret = -EBUSY; - goto out; - } - anon_vma_lock_write(anon_vma); - mapping = NULL; - } else { - unsigned int min_order; - gfp_t gfp; - - mapping = folio->mapping; - min_order = mapping_min_folio_order(mapping); - if (new_order < min_order) { - ret = -EINVAL; - goto out; - } - - gfp = current_gfp_context(mapping_gfp_mask(mapping) & - GFP_RECLAIM_MASK); - - if (!filemap_release_folio(folio, gfp)) { - ret = -EBUSY; - goto out; - } - - mapping_set_update(&xas, mapping); - - if (split_type == SPLIT_TYPE_UNIFORM) { - xas_set_order(&xas, folio->index, new_order); - xas_split_alloc(&xas, folio, old_order, gfp); - if (xas_error(&xas)) { - ret = xas_error(&xas); - goto out; - } - } - - anon_vma = NULL; - i_mmap_lock_read(mapping); - - /* - *__split_unmapped_folio() may need to trim off pages beyond - * EOF: but on 32-bit, i_size_read() takes an irq-unsafe - * seqlock, which cannot be nested inside the page tree lock. - * So note end now: i_size itself may be changed at any moment, - * but folio lock is good enough to serialize the trimming. - */ - end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); - if (shmem_mapping(mapping)) - end = shmem_fallocend(mapping->host, end); - } - - /* - * Racy check if we can split the page, before unmap_folio() will - * split PMDs - */ - if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) { - ret = -EAGAIN; - goto out_unlock; - } - - unmap_folio(folio); - - /* block interrupt reentry in xa_lock and spinlock */ - local_irq_disable(); - if (mapping) { - /* - * Check if the folio is present in page cache. - * We assume all tail are present too, if folio is there. - */ - xas_lock(&xas); - xas_reset(&xas); - if (xas_load(&xas) != folio) { - ret = -EAGAIN; - goto fail; - } - } - - ret = __folio_freeze_and_split_unmapped(folio, new_order, split_at, &xas, mapping, - true, list, split_type, end, &nr_shmem_dropped); -fail: - if (mapping) - xas_unlock(&xas); - - local_irq_enable(); - - if (nr_shmem_dropped) - shmem_uncharge(mapping->host, nr_shmem_dropped); - - if (!ret && is_anon && !folio_is_device_private(folio)) - ttu_flags = TTU_USE_SHARED_ZEROPAGE; - - remap_page(folio, 1 << old_order, ttu_flags); - - /* - * Drop the mapping while the inode is still pinned. @folio stays - * locked and present in the page cache until the loop below, so - * eviction cannot free the inode yet; @lock_at is not enough, it may - * be a tail beyond EOF that the split already dropped from the page - * cache. Nothing past this point may touch the inode or the mapping. - */ - if (mapping) { - i_mmap_unlock_read(mapping); - mapping = NULL; - } + if (is_anon) + ret = __folio_freeze_split_anon(folio, new_order, split_at, + true, list, split_type); + else + ret = __folio_freeze_split_file(folio, new_order, split_at, + list, split_type); /* * Unlock all after-split folios except the one containing @@ -4309,29 +4349,19 @@ fail: if (new_folio == page_folio(lock_at)) continue; - folio_unlock(new_folio); /* * Subpages whose mapping has been zapped may be freed * earlier, but freeing them requires taking the - * lru_lock, so we defer put_page() on tail pages until + * lru_lock, so we defer folio_put() on tail pages until * after the split completes. */ - free_folio_and_swap_cache(new_folio); + if (is_swapcache && !folio_mapped(new_folio)) + folio_free_swap(new_folio); + folio_unlock(new_folio); + folio_put(new_folio); } -out_unlock: - if (anon_vma) { - anon_vma_unlock_write(anon_vma); - put_anon_vma(anon_vma); - } - if (mapping) - i_mmap_unlock_read(mapping); out: - /* restore to caller's old_memcg */ - set_active_memcg(old_memcg); - mem_cgroup_put(memcg); -out_no_memcg: - xas_destroy(&xas); if (is_pmd_order(old_order)) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); @@ -4355,29 +4385,20 @@ out_no_memcg: * isolated from LRU (if applicable) * * Upon return, the folio is not remapped, split folios are not added to LRU, - * free_folio_and_swap_cache() is not called, and new folios remain locked. + * folio_free_swap() is not called, and new folios remain locked. * * Return: 0 on success, -EAGAIN if the folio cannot be split (e.g., due to * insufficient reference count or extra pins). */ int folio_split_unmapped(struct folio *folio, unsigned int new_order) { - int ret = 0; - VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_anon(folio), folio); - if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) - return -EAGAIN; - - local_irq_disable(); - ret = __folio_freeze_and_split_unmapped(folio, new_order, &folio->page, NULL, - NULL, false, NULL, SPLIT_TYPE_UNIFORM, - 0, NULL); - local_irq_enable(); - return ret; + return __folio_freeze_split_anon(folio, new_order, &folio->page, + false, NULL, SPLIT_TYPE_UNIFORM); } /* @@ -4528,11 +4549,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio) memcg = folio_memcg(folio); lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags); if (__list_lru_del(&deferred_split_lru, lru, &folio->_deferred_list, nid)) { - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(folio_order(folio), - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } + folio_reset_partially_mapped(folio); unqueued = true; } list_lru_unlock_irqrestore(lru, &flags); @@ -4634,22 +4651,11 @@ static enum lru_status deferred_split_isolate(struct list_head *item, struct folio *folio = container_of(item, struct folio, _deferred_list); struct list_head *freeable = cb_arg; - if (folio_try_get(folio)) { - list_lru_isolate_move(lru, item, freeable); - return LRU_REMOVED; - } + /* Lost race to folio_put() or the folio is under folio_ref_freeze() */ + if (!folio_try_get(folio)) + return LRU_SKIP; - /* - * We lost race with folio_put(). Read folio state before the - * isolate: folio_unqueue_deferred_split() checks list_empty() - * locklessly, so once removed the folio can be freed any time. - */ - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(folio_order(folio), - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } - list_lru_isolate(lru, item); + list_lru_isolate_move(lru, item, freeable); return LRU_REMOVED; } @@ -4854,7 +4860,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, * will try to drop it before split and then check if the folio * can be split or not. So skip the check here. */ - if (!folio_test_private(folio) && + if (!folio_has_attached_private(folio) && folio_expected_ref_count(folio) != folio_ref_count(folio)) goto next; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index cea25773a6c9..b017ae1dfd6f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -54,12 +55,20 @@ #include "mm_init.h" #include +#define HUGE_BOOTMEM_ZONES_VALID BIT(0) + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; +}; + int hugetlb_max_hstate __read_mostly; unsigned int default_hstate_idx; struct hstate hstates[HUGE_MAX_HSTATE]; __initdata nodemask_t hugetlb_bootmem_nodes; -__initdata struct list_head huge_boot_pages[MAX_NUMNODES]; +static struct list_head huge_boot_pages[MAX_NUMNODES] __initdata; /* * Due to ordering constraints across the init code for various @@ -116,6 +125,7 @@ struct mutex *hugetlb_fault_mutex_table __ro_after_init; /* Forward declaration */ static int hugetlb_acct_memory(struct hstate *h, long delta); +static unsigned int allowed_mems_nr(struct hstate *h); static void hugetlb_vma_lock_free(struct vm_area_struct *vma); static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma); @@ -1387,8 +1397,11 @@ void remove_hugetlb_folio(struct hstate *h, struct folio *folio, VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio); lockdep_assert_held(&hugetlb_lock); - if (hstate_is_gigantic_no_runtime(h)) + if (hstate_is_gigantic_no_runtime(h)) { + /* Callers must filter gigantic_no_runtime upstream. */ + VM_WARN_ON_ONCE(1); return; + } list_del(&folio->lru); @@ -1433,11 +1446,8 @@ void add_hugetlb_folio(struct hstate *h, struct folio *folio, } __folio_set_hugetlb(folio); - folio_change_private(folio, NULL); - /* - * We have to set hugetlb_vmemmap_optimized again as above - * folio_change_private(folio, NULL) cleared it. - */ + /* Clear all folio->private flags except hugetlb_vmemmap_optimized. */ + folio->private = NULL; folio_set_hugetlb_vmemmap_optimized(folio); arch_clear_hugetlb_flags(folio); @@ -1449,8 +1459,11 @@ static void __update_and_free_hugetlb_folio(struct hstate *h, { bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio); - if (hstate_is_gigantic_no_runtime(h)) + if (hstate_is_gigantic_no_runtime(h)) { + /* Callers must filter gigantic_no_runtime upstream. */ + VM_WARN_ON_ONCE(1); return; + } /* * If we don't know which subpages are hwpoisoned, we can't free @@ -2187,8 +2200,10 @@ struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid, folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, preferred_nid, nmask); - if (folio) + if (folio) { + folio_set_hugetlb_restore_reserve(folio); h->resv_huge_pages--; + } spin_unlock_irq(&hugetlb_lock); return folio; @@ -2235,6 +2250,19 @@ static nodemask_t *policy_mbind_nodemask(gfp_t gfp) return NULL; } +/* + * Reservations are globally accounted, but they must also be backed by free + * pages on nodes allowed by the current cpuset and MPOL_BIND policy. + */ +static long surplus_pages_needed(struct hstate *h, long delta, long allocated) +{ + long global_free = (long)h->free_huge_pages + allocated; + long allowed_free = (long)allowed_mems_nr(h) + allocated; + + return max((long)h->resv_huge_pages + delta - global_free, + delta - allowed_free); +} + /* * Increase the hugetlb pool such that it can accommodate a reservation * of size 'delta'. @@ -2257,7 +2285,7 @@ static int gather_surplus_pages(struct hstate *h, long delta) alloc_nodemask = cpuset_current_mems_allowed; lockdep_assert_held(&hugetlb_lock); - needed = (h->resv_huge_pages + delta) - h->free_huge_pages; + needed = surplus_pages_needed(h, delta, 0); if (needed <= 0) { h->resv_huge_pages += delta; return 0; @@ -2288,11 +2316,10 @@ retry: /* * After retaking hugetlb_lock, we need to recalculate 'needed' - * because either resv_huge_pages or free_huge_pages may have changed. + * because either resv_huge_pages or the free page counts may have changed. */ spin_lock_irq(&hugetlb_lock); - needed = (h->resv_huge_pages + delta) - - (h->free_huge_pages + allocated); + needed = surplus_pages_needed(h, delta, allocated); if (needed > 0) { if (alloc_ok) goto retry; @@ -2827,6 +2854,7 @@ void wait_for_freed_hugetlb_folios(void) * hugetlb_alloc_folio - Allocate a hugetlb folio. * @h: Hugetlb state control block. * @mpoli: Interpreted memory policy to use for allocation. + * @mm: Memory descriptor of the allocation target. * @alloc_flags: Flags controlling the allocation behavior. * * Allocates a hugetlb folio and handles cgroup charging and global hstate @@ -2834,10 +2862,10 @@ void wait_for_freed_hugetlb_folios(void) * * Return: A pointer to the allocated folio, or an ERR_PTR on failure. * -ENOSPC if cgroup charging fails or no folio is available. - * -ENOMEM if mem cgroup charging fails. */ struct folio *hugetlb_alloc_folio(struct hstate *h, - struct mempolicy_interpreted *mpoli, u8 alloc_flags) + struct mempolicy_interpreted *mpoli, struct mm_struct *mm, + u8 alloc_flags) { bool charge_hugetlb_cgroup_rsvd = alloc_flags & HUGETLB_ALLOC_CHARG_CGROUP_RSVD; @@ -2892,7 +2920,8 @@ struct folio *hugetlb_alloc_folio(struct hstate *h, spin_unlock_irq(&hugetlb_lock); - ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); + ret = mem_cgroup_charge_hugetlb(folio, mm, + gfp | __GFP_RETRY_MAYFAIL); /* * Unconditionally increment NR_HUGETLB here because if * mem_cgroup_charge_hugetlb failed, freeing the page will @@ -2901,13 +2930,17 @@ struct folio *hugetlb_alloc_folio(struct hstate *h, lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages); if (ret == -ENOMEM) { - free_huge_folio(folio); + folio_put(folio); /* * Skip uncharging hugetlb_cgroup since the charges * were committed to the folio and freeing the folio * would have cleared those up. */ - return ERR_PTR(ret); + /* + * Return -ENOSPC, since retrying the fault is futile: + * the OOM killer is not triggered for HugeTLB. + */ + return ERR_PTR(-ENOSPC); } return folio; @@ -3031,7 +3064,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, .nodemask = nodemask, }; - folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags); + folio = hugetlb_alloc_folio(h, &mpoli, vma->vm_mm, alloc_flags); mpol_cond_put(mpol); @@ -3130,7 +3163,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) */ INIT_LIST_HEAD(&m->list); m->hstate = h; - m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0; + m->flags = 0; /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */ if (!hugetlb_early_cma(h) && @@ -3148,6 +3181,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) } else { list_add_tail(&m->list, &huge_boot_pages[nid]); m->flags |= HUGE_BOOTMEM_ZONES_VALID; + hugetlb_vmemmap_optimize_bootmem_page(pfn, huge_page_order(h)); /* * Only initialize the head struct page in memmap_init_reserved_pages, * rest of the struct pages will be initialized by the HugeTLB @@ -3205,16 +3239,6 @@ static void __init hugetlb_folio_init_vmemmap(struct folio *folio, prep_compound_head(&folio->page, huge_page_order(h)); } -static bool __init hugetlb_bootmem_page_prehvo(struct huge_bootmem_page *m) -{ - return m->flags & HUGE_BOOTMEM_HVO; -} - -static bool __init hugetlb_bootmem_page_earlycma(struct huge_bootmem_page *m) -{ - return m->flags & HUGE_BOOTMEM_CMA; -} - /* * memblock-allocated pageblocks might not have the migrate type set * if marked with the 'noinit' flag. Set it to the default (MIGRATE_MOVABLE) @@ -3284,6 +3308,7 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) { struct page *page = virt_to_page(m); struct folio *folio = (void *)page; + const unsigned long pfn = folio_pfn(folio); h = m->hstate; /* @@ -3301,16 +3326,9 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) HUGETLB_VMEMMAP_RESERVE_PAGES); init_new_hugetlb_folio(folio); - if (hugetlb_bootmem_page_prehvo(m)) - /* - * If pre-HVO was done, just set the - * flag, the HVO code will then skip - * this folio. - */ + if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn))) folio_set_hugetlb_vmemmap_optimized(folio); - - if (hugetlb_bootmem_page_earlycma(m)) - folio_set_hugetlb_cma(folio); + section_set_compound_order_range(pfn, folio_nr_pages(folio), 0); list_add(&folio->lru, &folio_list); @@ -3322,7 +3340,9 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) * For CMA pages, this is done in init_cma_pageblock * (via hugetlb_bootmem_init_migratetype), so skip it here. */ - if (!folio_test_hugetlb_cma(folio)) + if (hugetlb_early_cma(h)) + folio_set_hugetlb_cma(folio); + else adjust_managed_page_count(page, pages_per_huge_page(h)); cond_resched(); } @@ -3351,31 +3371,6 @@ void __init hugetlb_bootmem_struct_page_init(void) .max_threads = num_node_state(N_MEMORY), .numa_aware = true, }; -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP - struct zone *zone; - - for_each_zone(zone) { - for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { - struct page *tail, *p; - unsigned int order; - - tail = zone->vmemmap_tails[i]; - if (!tail) - continue; - - order = i + VMEMMAP_TAIL_MIN_ORDER; - p = page_to_virt(tail); - /* - * prep_and_add_bootmem_folios() can access pageblock - * flags on bootmem HugeTLB pages, so initialize the - * shared tail struct pages here before bootmem folios - * start using them. - */ - for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) - init_compound_tail(p + j, NULL, order, zone); - } - } -#endif padata_do_multithreaded(&job); } @@ -3425,7 +3420,7 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, &node_states[N_MEMORY], NULL); if (!folio && !list_empty(&folio_list) && - hugetlb_vmemmap_optimizable_size(h)) { + hugetlb_vmemmap_optimizable(h)) { prep_and_add_allocated_folios(h, &folio_list); INIT_LIST_HEAD(&folio_list); folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, @@ -3494,7 +3489,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l for (i = 0; i < num; ++i) { struct folio *folio; - if (hugetlb_vmemmap_optimizable_size(h) && + if (hugetlb_vmemmap_optimizable(h) && (si_mem_available() == 0) && !list_empty(&folio_list)) { prep_and_add_allocated_folios(h, &folio_list); INIT_LIST_HEAD(&folio_list); @@ -4021,6 +4016,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, struct hstate *dst; long rc = 0; long nr_demoted = 0; + long nr_persistent = 0; lockdep_assert_held(&hugetlb_lock); @@ -4033,22 +4029,60 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, for_each_node_mask_to_free(src, nr_nodes, node, nodes_allowed) { LIST_HEAD(list); + LIST_HEAD(surplus_list); struct folio *folio, *next; + unsigned long nr_available, nr_target; + + /* + * Re-check available each node batch: the previous + * batch released hugetlb_lock for vmemmap restore/split, + * and a new reservation could have been added in that + * window, shrinking the budget. available is global + * (resv is not per-node), so 0 means no node can + * contribute -- stop the whole scan. + */ + nr_available = available_huge_pages(src); + if (!nr_available) + break; + + /* + * Cap this batch at the current budget; expressed as a + * cumulative stop point because nr_demoted is running. + */ + nr_target = nr_demoted + min_t(unsigned long, + nr_to_demote - nr_demoted, nr_available); list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) { + bool adjust_surplus; + if (folio_test_hwpoison(folio)) continue; - remove_hugetlb_folio(src, folio, false); - list_add(&folio->lru, &list); + /* Surplus accounting is maintained per node, not per folio. */ + adjust_surplus = src->surplus_huge_pages_node[node] > 0; + remove_hugetlb_folio(src, folio, adjust_surplus); + list_add(&folio->lru, adjust_surplus ? &surplus_list : &list); + if (!adjust_surplus) + nr_persistent++; - if (++nr_demoted == nr_to_demote) + if (++nr_demoted == nr_target) break; } + if (list_empty(&list) && list_empty(&surplus_list)) + continue; + spin_unlock_irq(&hugetlb_lock); - rc = demote_free_hugetlb_folios(src, dst, &list); + if (!list_empty(&list)) + rc = demote_free_hugetlb_folios(src, dst, &list); + if (!list_empty(&surplus_list)) { + long tmp_rc; + + tmp_rc = demote_free_hugetlb_folios(src, dst, &surplus_list); + if (rc >= 0) + rc = tmp_rc; + } spin_lock_irq(&hugetlb_lock); @@ -4056,6 +4090,14 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, list_del(&folio->lru); add_hugetlb_folio(src, folio, false); + nr_demoted--; + nr_persistent--; + } + + list_for_each_entry_safe(folio, next, &surplus_list, lru) { + list_del(&folio->lru); + add_hugetlb_folio(src, folio, true); + nr_demoted--; } @@ -4067,7 +4109,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, * Not absolutely necessary, but for consistency update max_huge_pages * based on pool changes for the demoted page. */ - src->max_huge_pages -= nr_demoted; + src->max_huge_pages -= nr_persistent; dst->max_huge_pages += nr_demoted << (huge_page_order(src) - huge_page_order(dst)); if (rc < 0) @@ -4277,9 +4319,9 @@ static __init void hugetlb_parse_params(void) /* * hugepages command line processing - * hugepages normally follows a valid hugepagsz or default_hugepagsz - * specification. If not, ignore the hugepages value. hugepages can also - * be the first huge page command line option in which case it implicitly + * hugepages normally follows a valid hugepagesz or default_hugepagesz + * specification. If not, ignore the hugepages value. hugepages can also + * be the first huge page command line option in which case it implicitly * specifies the number of huge pages for the default size. */ static int __init hugepages_setup(char *s) @@ -6822,15 +6864,6 @@ long hugetlb_reserve_pages(struct inode *inode, out_put_pages: spool_resv = chg - gbl_reserve; - if (spool_resv) { - /* put sub pool's reservation back, chg - gbl_reserve */ - gbl_resv = hugepage_subpool_put_pages(spool, spool_resv); - /* - * subpool's reserved pages can not be put back due to race, - * return to hstate. - */ - hugetlb_acct_memory(h, -gbl_resv); - } /* Restore used_hpages for pages that failed global reservation */ if (gbl_reserve && spool) { unsigned long flags; @@ -6840,6 +6873,15 @@ out_put_pages: spool->used_hpages -= gbl_reserve; unlock_or_release_subpool(spool, flags); } + if (spool_resv) { + /* put sub pool's reservation back, chg - gbl_reserve */ + gbl_resv = hugepage_subpool_put_pages(spool, spool_resv); + /* + * subpool's reserved pages can not be put back due to race, + * return to hstate. + */ + hugetlb_acct_memory(h, -gbl_resv); + } out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), chg * pages_per_huge_page(h), h_cg); diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index ecb6e0b7819a..3fb41311e4c7 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -179,6 +179,34 @@ static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) hugetlb_cgroup_free(hugetlb_cgroup_from_css(css)); } +static void hugetlb_cgroup_move_usage(struct hugetlb_cgroup *from, + struct hugetlb_cgroup *to, + struct folio *from_folio, + struct folio *to_folio) +{ + int idx = hstate_index(folio_hstate(from_folio)); + unsigned long nr_pages = folio_nr_pages(from_folio); + int from_nid = folio_nid(from_folio); + int to_nid = folio_nid(to_folio); + unsigned long usage; + + lockdep_assert_held(&hugetlb_lock); + + if (!from || !to) + return; + + if (from == to && from_nid == to_nid) + return; + + usage = from->nodeinfo[from_nid]->usage[idx]; + if (WARN_ON_ONCE(usage < nr_pages)) + return; + WRITE_ONCE(from->nodeinfo[from_nid]->usage[idx], usage - nr_pages); + + usage = to->nodeinfo[to_nid]->usage[idx]; + WRITE_ONCE(to->nodeinfo[to_nid]->usage[idx], usage + nr_pages); +} + /* * Should be called with hugetlb_lock held. * Since we are holding hugetlb_lock, pages cannot get moved from @@ -213,6 +241,8 @@ static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg, /* Take the pages off the local counter */ page_counter_cancel(counter, nr_pages); + hugetlb_cgroup_move_usage(h_cg, parent, folio, folio); + set_hugetlb_cgroup(folio, parent); out: return; @@ -906,6 +936,9 @@ void hugetlb_cgroup_migrate(struct folio *old_folio, struct folio *new_folio) /* move the h_cg details to new cgroup */ set_hugetlb_cgroup(new_folio, h_cg); set_hugetlb_cgroup_rsvd(new_folio, h_cg_rsvd); + + hugetlb_cgroup_move_usage(h_cg, h_cg, old_folio, new_folio); + list_move(&new_folio->lru, &h->hugepage_activelist); spin_unlock_irq(&hugetlb_lock); } diff --git a/mm/hugetlb_sysfs.c b/mm/hugetlb_sysfs.c index 79ece91406bf..326a54b4d991 100644 --- a/mm/hugetlb_sysfs.c +++ b/mm/hugetlb_sysfs.c @@ -211,15 +211,15 @@ static ssize_t demote_store(struct kobject *kobj, * Check for available pages to demote each time thorough the * loop as demote_pool_huge_page will drop hugetlb_lock. */ + nr_available = h->free_huge_pages - h->resv_huge_pages; if (nid != NUMA_NO_NODE) - nr_available = h->free_huge_pages_node[nid]; - else - nr_available = h->free_huge_pages; - nr_available -= h->resv_huge_pages; + nr_available = min(nr_available, + h->free_huge_pages_node[nid]); if (!nr_available) break; - rc = demote_pool_huge_page(h, n_mask, nr_demote); + rc = demote_pool_huge_page(h, n_mask, + min(nr_demote, nr_available)); if (rc < 0) { err = rc; break; diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 917db0984143..0057fa2a16a8 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -15,10 +15,10 @@ #include #include #include +#include #include #include "hugetlb_vmemmap.h" -#include "internal.h" /** * struct vmemmap_remap_walk - walk vmemmap page table @@ -492,32 +492,6 @@ static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio * return true; } -static struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) -{ - const unsigned int idx = order - VMEMMAP_TAIL_MIN_ORDER; - struct page *tail, *p; - int node = zone_to_nid(zone); - - tail = READ_ONCE(zone->vmemmap_tails[idx]); - if (likely(tail)) - return tail; - - tail = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); - if (!tail) - return NULL; - - p = page_to_virt(tail); - for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) - init_compound_tail(p + i, NULL, order, zone); - - if (cmpxchg(&zone->vmemmap_tails[idx], NULL, tail)) { - __free_page(tail); - tail = READ_ONCE(zone->vmemmap_tails[idx]); - } - - return tail; -} - static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio, struct list_head *vmemmap_pages, @@ -534,7 +508,7 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, return ret; nid = folio_nid(folio); - vmemmap_tail = vmemmap_get_tail(h->order, folio_zone(folio)); + vmemmap_tail = vmemmap_shared_tail_page(h->order, folio_zone(folio)); if (!vmemmap_tail) return -ENOMEM; @@ -705,109 +679,13 @@ void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head __hugetlb_vmemmap_optimize_folios(h, folio_list, true); } -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT - -/* Return true of a bootmem allocated HugeTLB page should be pre-HVO-ed */ -static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m) +void __init hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order) { - unsigned long section_size, psize, pmd_vmemmap_size; - phys_addr_t paddr; - - if (!READ_ONCE(vmemmap_optimize_enabled)) - return false; - - if (!hugetlb_vmemmap_optimizable(m->hstate)) - return false; - - psize = huge_page_size(m->hstate); - paddr = virt_to_phys(m); - - /* - * Pre-HVO only works if the bootmem huge page - * is aligned to the section size. - */ - section_size = (1UL << PA_SECTION_SHIFT); - if (!IS_ALIGNED(paddr, section_size) || - !IS_ALIGNED(psize, section_size)) - return false; - - /* - * The pre-HVO code does not deal with splitting PMDS, - * so the bootmem page must be aligned to the number - * of base pages that can be mapped with one vmemmap PMD. - */ - pmd_vmemmap_size = (PMD_SIZE / (sizeof(struct page))) << PAGE_SHIFT; - if (!IS_ALIGNED(paddr, pmd_vmemmap_size) || - !IS_ALIGNED(psize, pmd_vmemmap_size)) - return false; - - return true; -} - -static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) -{ - struct zone *zone; - enum zone_type zone_type; - - for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { - zone = &NODE_DATA(nid)->node_zones[zone_type]; - if (zone_spans_pfn(zone, pfn)) - return zone; - } - - return NULL; -} - -/* - * Initialize memmap section for a gigantic page, HVO-style. - */ -void __init hugetlb_vmemmap_init_early(int nid) -{ - unsigned long psize, paddr, section_size; - unsigned long ns, i, pnum, pfn, nr_pages; - unsigned long start, end; - struct huge_bootmem_page *m = NULL; - void *map; - if (!READ_ONCE(vmemmap_optimize_enabled)) return; - section_size = (1UL << PA_SECTION_SHIFT); - - list_for_each_entry(m, &huge_boot_pages[nid], list) { - struct zone *zone; - - if (!vmemmap_should_optimize_bootmem_page(m)) - continue; - - nr_pages = pages_per_huge_page(m->hstate); - psize = nr_pages << PAGE_SHIFT; - paddr = virt_to_phys(m); - pfn = PHYS_PFN(paddr); - map = pfn_to_page(pfn); - start = (unsigned long)map; - end = start + hugetlb_vmemmap_size(m->hstate); - zone = pfn_to_zone(nid, pfn); - - if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate), - zone, HUGETLB_VMEMMAP_RESERVE_SIZE)) - panic("Failed to allocate memmap for HugeTLB page\n"); - memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE)); - - pnum = pfn_to_section_nr(pfn); - ns = psize / section_size; - - for (i = 0; i < ns; i++) { - sparse_init_early_section(nid, map, pnum, - SECTION_IS_VMEMMAP_PREINIT); - map += section_map_size(); - pnum++; - } - - m->flags |= HUGE_BOOTMEM_HVO; - } + section_set_compound_order_range(pfn, 1UL << order, order); } -#endif static const struct ctl_table hugetlb_vmemmap_sysctls[] = { { diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 7ac49c52457d..464192e32dec 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -9,8 +9,6 @@ #ifndef _LINUX_HUGETLB_VMEMMAP_H #define _LINUX_HUGETLB_VMEMMAP_H #include -#include -#include /* * Reserve one vmemmap page, all vmemmap addresses are mapped to it. See @@ -27,10 +25,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h, void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list); void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -void hugetlb_vmemmap_init_early(int nid); -#endif - +void hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order); static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h) { @@ -76,14 +71,14 @@ static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, { } -static inline void hugetlb_vmemmap_init_early(int nid) -{ -} - static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h) { return 0; } + +static inline void hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order) +{ +} #endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */ static inline bool hugetlb_vmemmap_optimizable(const struct hstate *h) diff --git a/mm/init-mm.c b/mm/init-mm.c index 3e792aad7626..a1bb2c2d0284 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -39,10 +39,8 @@ struct mm_struct init_mm = { .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist = LIST_HEAD_INIT(init_mm.mmlist), -#ifdef CONFIG_PER_VMA_LOCK .vma_writer_wait = __RCUWAIT_INITIALIZER(init_mm.vma_writer_wait), .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), -#endif #ifdef CONFIG_SCHED_MM_CID .mm_cid.lock = __RAW_SPIN_LOCK_UNLOCKED(init_mm.mm_cid.lock), #endif diff --git a/mm/internal.h b/mm/internal.h index 38b1165212c9..05179c4b2090 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,13 +23,6 @@ #include "vma.h" struct folio_batch; -struct hstate; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; -}; /* mm/workingset.c */ bool workingset_test_recent(void *shadow, bool file, bool *workingset, @@ -85,10 +78,6 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, gfp_t gfp_mask, unsigned int reclaim_options, int *swappiness); -unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned); #ifdef CONFIG_NUMA extern int sysctl_min_unmapped_ratio; @@ -237,15 +226,18 @@ static inline int mmap_file(struct file *file, struct vm_area_struct *vma) { int err = vfs_mmap(file, vma); - if (likely(!err)) - return 0; - /* - * OK, we tried to call the file hook for mmap(), but an error - * arose. The mapping is in an inconsistent state and we must not invoke - * any further hooks on it. + * Either we tried to call the file hook for mmap() and an error arose + * or a driver set vma->vm_ops = NULL intending there to be no VMA + * operations. + * + * In the former case the VMA is in an inconsistent state and we mustn't + * invoke any further hooks on it, in the latter case the hook actually + * wanted no further hooks to be invoked, so fix both by setting dummy + * VMA ops. */ - vma->vm_ops = &vma_dummy_vm_ops; + if (unlikely(err || !vma->vm_ops)) + vma->vm_ops = &vma_dummy_vm_ops; return err; } @@ -295,11 +287,6 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma) down_write(&anon_vma->root->rwsem); } -static inline int anon_vma_trylock_write(struct anon_vma *anon_vma) -{ - return down_write_trylock(&anon_vma->root->rwsem); -} - static inline void anon_vma_unlock_write(struct anon_vma *anon_vma) { up_write(&anon_vma->root->rwsem); @@ -799,15 +786,6 @@ static inline void prep_compound_tail(struct page *tail, VM_WARN_ON_ONCE(tail->private); } -static inline void init_compound_tail(struct page *tail, - const struct page *head, unsigned int order, struct zone *zone) -{ - atomic_set(&tail->_mapcount, -1); - set_page_node(tail, zone_to_nid(zone)); - set_page_zone(tail, zone_idx(zone)); - prep_compound_tail(tail, head, order); -} - #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* @@ -1146,7 +1124,8 @@ extern int node_reclaim_mode; extern unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order); -extern int find_next_best_node(int node, nodemask_t *used_node_mask); +int find_next_best_node_in(int node, nodemask_t *used_node_mask, + const nodemask_t *candidates); #else #define node_reclaim_mode 0 @@ -1155,7 +1134,8 @@ static inline unsigned long node_reclaim(struct pglist_data *pgdat, { return 0; } -static inline int find_next_best_node(int node, nodemask_t *used_node_mask) +static inline int find_next_best_node_in(int node, nodemask_t *used_node_mask, + const nodemask_t *candidates) { return NUMA_NO_NODE; } @@ -1654,4 +1634,7 @@ static inline bool can_spin_trylock(void) return true; } +/* char-mem.c */ +bool file_is_dev_zero(const struct file *file); + #endif /* __MM_INTERNAL_H */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 75639298efc2..85ea095906fb 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1278,6 +1278,23 @@ static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_stru return SCAN_SUCCEED; } +static pgtable_t alloc_deposit_pte_table(struct mm_struct *mm) +{ + /* + * khugepaged is run from a kernel thread, so need to manually set the + * correct memcg so the allocation gets charged correctly. + */ + struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); + struct mem_cgroup *old_memcg = set_active_memcg(memcg); + pgtable_t pgtable; + + pgtable = pte_alloc_one(mm); + + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); + return pgtable; +} + /* * collapse_huge_page() expects the mmap_lock to be unlocked before entering and * will always return with the lock unlocked, to avoid holding the mmap_lock @@ -1293,7 +1310,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s LIST_HEAD(compound_pagelist); pmd_t *pmd, _pmd; pte_t *pte = NULL; - pgtable_t pgtable; + pgtable_t pgtable = NULL; struct folio *folio; spinlock_t *pmd_ptl, *pte_ptl; enum scan_result result = SCAN_FAIL; @@ -1310,6 +1327,14 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s goto out_nolock; } + if (is_pmd_order(order)) { + pgtable = alloc_deposit_pte_table(mm); + if (!pgtable) { + result = SCAN_ALLOC_HUGE_PAGE_FAIL; + goto out_nolock; + } + } + mmap_read_lock(mm); result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true, &vma, cc, order); @@ -1433,8 +1458,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s spin_lock(pmd_ptl); VM_WARN_ON_ONCE(!pmd_none(*pmd)); if (is_pmd_order(order)) { - pgtable = pmd_pgtable(_pmd); pgtable_trans_huge_deposit(mm, pmd, pgtable); + pgtable = NULL; map_anon_folio_pmd_nopf(folio, pmd, vma, pmd_addr); } else { /* @@ -1453,6 +1478,9 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s } spin_unlock(pmd_ptl); + if (is_pmd_order(order)) + pte_free_defer(mm, pmd_pgtable(_pmd)); + folio = NULL; result = SCAN_SUCCEED; @@ -1463,6 +1491,8 @@ out_up_write: anon_vma_unlock_write(vma->anon_vma); mmap_write_unlock(mm); out_nolock: + if (pgtable) + pte_free(mm, pgtable); if (folio) folio_put(folio); trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result, order); @@ -1612,6 +1642,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, enum scan_result result = SCAN_FAIL; struct page *page = NULL; struct folio *folio = NULL; + unsigned long failed_pfn = -1; unsigned long addr; unsigned long enabled_orders; spinlock_t *ptl; @@ -1706,11 +1737,13 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, if (cc->is_khugepaged && !(vma->vm_flags & VM_DROPPABLE) && folio_test_lazyfree(folio) && !pte_dirty(pteval)) { result = SCAN_PAGE_LAZYFREE; + failed_pfn = folio_pfn(folio); goto out_unmap; } if (!folio_test_anon(folio)) { result = SCAN_PAGE_ANON; + failed_pfn = folio_pfn(folio); goto out_unmap; } @@ -1721,6 +1754,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, if (folio_maybe_mapped_shared(folio)) { if (++shared > max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; + failed_pfn = folio_pfn(folio); count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_SHARED_PTE, MTHP_STAT_COLLAPSE_EXCEED_SHARED); goto out_unmap; @@ -1738,15 +1772,18 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, node = folio_nid(folio); if (collapse_scan_abort(node, cc)) { result = SCAN_SCAN_ABORT; + failed_pfn = folio_pfn(folio); goto out_unmap; } cc->node_load[node]++; if (!folio_test_lru(folio)) { result = SCAN_PAGE_LRU; + failed_pfn = folio_pfn(folio); goto out_unmap; } if (folio_test_locked(folio)) { result = SCAN_PAGE_LOCK; + failed_pfn = folio_pfn(folio); goto out_unmap; } @@ -1759,6 +1796,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, */ if (folio_expected_ref_count(folio) != folio_ref_count(folio)) { result = SCAN_PAGE_COUNT; + failed_pfn = folio_pfn(folio); goto out_unmap; } @@ -1784,7 +1822,7 @@ out_unmap: *lock_dropped = true; } out: - trace_mm_khugepaged_scan_pmd(mm, folio, referenced, + trace_mm_khugepaged_scan_pmd(mm, failed_pfn, referenced, none_or_zero, result, unmapped); return result; } @@ -2245,6 +2283,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, struct address_space *mapping = file->f_mapping; struct page *dst; struct folio *folio, *tmp, *new_folio; + unsigned long new_pfn = -1; pgoff_t index = 0, end = start + HPAGE_PMD_NR; LIST_HEAD(pagelist); XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); @@ -2264,6 +2303,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER); if (result != SCAN_SUCCEED) goto out; + new_pfn = folio_pfn(new_folio); mapping_set_update(&xas, mapping); @@ -2667,7 +2707,7 @@ rollback: folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result); + trace_mm_khugepaged_collapse_file(mm, new_pfn, index, addr, is_shmem, file, HPAGE_PMD_NR, result); return result; } @@ -2683,6 +2723,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, int present, swap; int node = NUMA_NO_NODE; enum scan_result result = SCAN_SUCCEED; + unsigned long failed_pfn = -1; present = 0; swap = 0; @@ -2715,6 +2756,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, if (is_pmd_order(folio_order(folio))) { result = SCAN_PTE_MAPPED_HUGEPAGE; + failed_pfn = folio_pfn(folio); /* * PMD-sized THP implies that we can only try * retracting the PTE table. @@ -2726,6 +2768,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, node = folio_nid(folio); if (collapse_scan_abort(node, cc)) { result = SCAN_SCAN_ABORT; + failed_pfn = folio_pfn(folio); folio_put(folio); break; } @@ -2733,12 +2776,14 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, if (!folio_test_lru(folio)) { result = SCAN_PAGE_LRU; + failed_pfn = folio_pfn(folio); folio_put(folio); break; } if (folio_expected_ref_count(folio) + 1 != folio_ref_count(folio)) { result = SCAN_PAGE_COUNT; + failed_pfn = folio_pfn(folio); folio_put(folio); break; } @@ -2773,7 +2818,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, } } - trace_mm_khugepaged_scan_file(mm, folio, file, present, swap, result); + trace_mm_khugepaged_scan_file(mm, failed_pfn, file, present, swap, result); return result; } diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 8fa409a4f9fb..20c0eb41c873 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -238,7 +238,7 @@ static struct task_struct *scan_thread; static unsigned long jiffies_min_age; /* consecutive scans an object must stay unreferenced before reporting */ static unsigned int min_unref_scans = - IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 2 : 1; + IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 3 : 1; module_param(min_unref_scans, uint, 0644); static unsigned long jiffies_last_scan; /* delay between automatic memory scannings */ @@ -1855,6 +1855,52 @@ static void dedup_flush(struct xarray *dedup) } } +/* + * Scan the struct pages of a zone, skipping memory holes, pages that belong to + * another zone and pages that are not in use. Returns 1 if the scan should be + * stopped. + */ +static int scan_zone_pages(struct zone *zone) +{ + const unsigned int max_batch = MAX_SCAN_SIZE / sizeof(struct page); + unsigned long start_pfn = zone->zone_start_pfn; + unsigned long end_pfn = zone_end_pfn(zone); + struct page *first = NULL, *last = NULL; + unsigned int batch = 0; + unsigned long pfn; + + for (pfn = start_pfn; pfn < end_pfn; pfn++) { + struct page *page = pfn_to_online_page(pfn); + + if (!(pfn & 63)) + cond_resched_tasks_rcu_qs(); + + /* only scan in-use pages belonging to this zone */ + if (page && (page_zone(page) != zone || + page_count(page) == 0)) + page = NULL; + + if (page && first && page == last + 1 && + batch < max_batch) { + last = page; + batch++; + continue; + } + + if (first && scan_block(first, last + 1, NULL)) + return 1; + + first = page; + last = page; + batch = page ? 1 : 0; + } + + if (first && scan_block(first, last + 1, NULL)) + return 1; + + return 0; +} + /* * Scan data sections and all the referenced memory blocks allocated via the * kernel's standard allocators. This function must be called with the @@ -1928,29 +1974,7 @@ static int __kmemleak_scan(bool full) */ get_online_mems(); for_each_populated_zone(zone) { - unsigned long start_pfn = zone->zone_start_pfn; - unsigned long end_pfn = zone_end_pfn(zone); - unsigned long pfn; - - for (pfn = start_pfn; pfn < end_pfn; pfn++) { - struct page *page = pfn_to_online_page(pfn); - - if (!(pfn & 63)) - cond_resched_tasks_rcu_qs(); - - if (!page) - continue; - - /* only scan pages belonging to this zone */ - if (page_zone(page) != zone) - continue; - /* only scan if page is in use */ - if (page_count(page) == 0) - continue; - stop = scan_block(page, page + 1, NULL); - if (stop) - break; - } + stop = scan_zone_pages(zone); if (stop) break; } diff --git a/mm/ksm.c b/mm/ksm.c index 49d48d1e0998..624f37975e12 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -348,7 +348,7 @@ static enum ksm_advisor_type ksm_advisor; * Only called through the sysfs control interface: */ -/* At least scan this many pages per batch. */ +/* Initial number of pages to scan per batch. */ static unsigned long ksm_advisor_min_pages_to_scan = 500; static void set_advisor_defaults(void) @@ -1115,7 +1115,8 @@ static inline void folio_set_stable_node(struct folio *folio, struct ksm_stable_node *stable_node) { VM_WARN_ON_FOLIO(folio_test_anon(folio) && PageAnonExclusive(&folio->page), folio); - folio->mapping = (void *)((unsigned long)stable_node | FOLIO_MAPPING_KSM); + WRITE_ONCE(folio->mapping, + (void *)((unsigned long)stable_node | FOLIO_MAPPING_KSM)); } #ifdef CONFIG_SYSFS @@ -3316,7 +3317,7 @@ void folio_migrate_ksm(struct folio *newfolio, struct folio *folio) stable_node = folio_stable_node(folio); if (stable_node) { VM_BUG_ON_FOLIO(stable_node->kpfn != folio_pfn(folio), folio); - stable_node->kpfn = folio_pfn(newfolio); + WRITE_ONCE(stable_node->kpfn, folio_pfn(newfolio)); /* * newfolio->mapping was set in advance; now we need smp_wmb() * to make sure that the new stable_node->kpfn is visible diff --git a/mm/list_lru.c b/mm/list_lru.c index 36662d02ff96..8a6dd0a489e1 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -666,12 +666,17 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, struct shrinker *shr int i; #ifdef CONFIG_MEMCG - if (shrinker) + /* + * If the shrinker fell back to being non-memcg-aware (e.g. with + * cgroup.memory=nokmem), its id was never assigned and holds a + * stale 0. Don't let set_shrinker_bit() act on it. + */ + if (shrinker && (shrinker->flags & SHRINKER_MEMCG_AWARE)) lru->shrinker_id = shrinker->id; else lru->shrinker_id = -1; - if (mem_cgroup_kmem_disabled()) + if (mem_cgroup_disabled() || mem_cgroup_kmem_disabled()) memcg_aware = false; #endif diff --git a/mm/madvise.c b/mm/madvise.c index eeee82cf2b3f..bef4de4425f2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -297,11 +297,12 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) loff_t offset; #ifdef CONFIG_SWAP - if (!file) { + if (vma_is_cow_mapping(vma) && vma->anon_vma) { walk_page_range_vma(vma, start, end, &swapin_walk_ops, vma); lru_add_drain(); /* Push any new pages onto the LRU now */ - return 0; } + if (!file) + return 0; if (shmem_mapping(file->f_mapping)) { shmem_swapin_range(vma, start, end, file->f_mapping); @@ -393,16 +394,18 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, return 0; orig_pmd = *pmd; - if (is_huge_zero_pmd(orig_pmd)) - goto huge_unlock; - if (unlikely(!pmd_present(orig_pmd))) { VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && !pmd_is_device_private_entry(orig_pmd)); goto huge_unlock; } - folio = pmd_folio(orig_pmd); + folio = vm_normal_folio_pmd(vma, addr, orig_pmd); + if (!folio) + goto huge_unlock; + + if (folio_is_zone_device(folio)) + goto huge_unlock; /* Do not interfere with other mappings of this folio */ if (folio_maybe_mapped_shared(folio)) @@ -414,9 +417,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (next - addr != HPAGE_PMD_SIZE) { int err; + if (!folio_trylock(folio)) + goto huge_unlock; folio_get(folio); spin_unlock(ptl); - folio_lock(folio); err = split_folio(folio); folio_unlock(folio); folio_put(folio); @@ -459,7 +463,7 @@ regular_folio: restart: start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!start_pte) - return 0; + goto out; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { @@ -563,6 +567,7 @@ restart: folio_deactivate(folio); } +out: if (start_pte) { lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 05ef55cae4dc..bf2c7d53b01b 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -17,30 +17,6 @@ #include "swap_table.h" #include "memcontrol-v1.h" -/* - * Cgroups above their limits are maintained in a RB-Tree, independent of - * their hierarchy representation - */ - -struct mem_cgroup_tree_per_node { - struct rb_root rb_root; - struct rb_node *rb_rightmost; - spinlock_t lock; -}; - -struct mem_cgroup_tree { - struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; -}; - -static struct mem_cgroup_tree soft_limit_tree __read_mostly; - -/* - * Maximum loops in mem_cgroup_soft_reclaim(), used for soft - * limit reclaim to prevent infinite loops, if they ever occur. - */ -#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 -#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 - /* for OOM */ struct mem_cgroup_eventfd_list { struct list_head list; @@ -106,301 +82,6 @@ static struct lockdep_map memcg_oom_lock_dep_map = { DEFINE_SPINLOCK(memcg_oom_lock); -static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz, - struct mem_cgroup_tree_per_node *mctz, - unsigned long new_usage_in_excess) -{ - struct rb_node **p = &mctz->rb_root.rb_node; - struct rb_node *parent = NULL; - struct mem_cgroup_per_node *mz_node; - bool rightmost = true; - - if (mz->on_tree) - return; - - mz->usage_in_excess = new_usage_in_excess; - if (!mz->usage_in_excess) - return; - while (*p) { - parent = *p; - mz_node = rb_entry(parent, struct mem_cgroup_per_node, - tree_node); - if (mz->usage_in_excess < mz_node->usage_in_excess) { - p = &(*p)->rb_left; - rightmost = false; - } else { - p = &(*p)->rb_right; - } - } - - if (rightmost) - mctz->rb_rightmost = &mz->tree_node; - - rb_link_node(&mz->tree_node, parent, p); - rb_insert_color(&mz->tree_node, &mctz->rb_root); - mz->on_tree = true; -} - -static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, - struct mem_cgroup_tree_per_node *mctz) -{ - if (!mz->on_tree) - return; - - if (&mz->tree_node == mctz->rb_rightmost) - mctz->rb_rightmost = rb_prev(&mz->tree_node); - - rb_erase(&mz->tree_node, &mctz->rb_root); - mz->on_tree = false; -} - -static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, - struct mem_cgroup_tree_per_node *mctz) -{ - unsigned long flags; - - spin_lock_irqsave(&mctz->lock, flags); - __mem_cgroup_remove_exceeded(mz, mctz); - spin_unlock_irqrestore(&mctz->lock, flags); -} - -static unsigned long soft_limit_excess(struct mem_cgroup *memcg) -{ - unsigned long nr_pages = page_counter_read(&memcg->memory); - unsigned long soft_limit = READ_ONCE(memcg->soft_limit); - unsigned long excess = 0; - - if (nr_pages > soft_limit) - excess = nr_pages - soft_limit; - - return excess; -} - -static void memcg1_update_tree(struct mem_cgroup *memcg, int nid) -{ - unsigned long excess; - struct mem_cgroup_per_node *mz; - struct mem_cgroup_tree_per_node *mctz; - - if (lru_gen_enabled()) { - if (soft_limit_excess(memcg)) - lru_gen_soft_reclaim(memcg, nid); - return; - } - - mctz = soft_limit_tree.rb_tree_per_node[nid]; - if (!mctz) - return; - /* - * Necessary to update all ancestors when hierarchy is used. - * because their event counter is not touched. - */ - for (; memcg; memcg = parent_mem_cgroup(memcg)) { - mz = memcg->nodeinfo[nid]; - excess = soft_limit_excess(memcg); - /* - * We have to update the tree if mz is on RB-tree or - * mem is over its softlimit. - */ - if (excess || mz->on_tree) { - unsigned long flags; - - spin_lock_irqsave(&mctz->lock, flags); - /* if on-tree, remove it */ - if (mz->on_tree) - __mem_cgroup_remove_exceeded(mz, mctz); - /* - * Insert again. mz->usage_in_excess will be updated. - * If excess is 0, no tree ops. - */ - __mem_cgroup_insert_exceeded(mz, mctz, excess); - spin_unlock_irqrestore(&mctz->lock, flags); - } - } -} - -void memcg1_remove_from_trees(struct mem_cgroup *memcg) -{ - struct mem_cgroup_tree_per_node *mctz; - struct mem_cgroup_per_node *mz; - int nid; - - for_each_node(nid) { - mz = memcg->nodeinfo[nid]; - mctz = soft_limit_tree.rb_tree_per_node[nid]; - if (mctz) - mem_cgroup_remove_exceeded(mz, mctz); - } -} - -static struct mem_cgroup_per_node * -__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) -{ - struct mem_cgroup_per_node *mz; - -retry: - mz = NULL; - if (!mctz->rb_rightmost) - goto done; /* Nothing to reclaim from */ - - mz = rb_entry(mctz->rb_rightmost, - struct mem_cgroup_per_node, tree_node); - /* - * Remove the node now but someone else can add it back, - * we will to add it back at the end of reclaim to its correct - * position in the tree. - */ - __mem_cgroup_remove_exceeded(mz, mctz); - if (!soft_limit_excess(mz->memcg) || - !css_tryget(&mz->memcg->css)) - goto retry; -done: - return mz; -} - -static struct mem_cgroup_per_node * -mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) -{ - struct mem_cgroup_per_node *mz; - - spin_lock_irq(&mctz->lock); - mz = __mem_cgroup_largest_soft_limit_node(mctz); - spin_unlock_irq(&mctz->lock); - return mz; -} - -static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, - pg_data_t *pgdat, - gfp_t gfp_mask, - unsigned long *total_scanned) -{ - struct mem_cgroup *victim = NULL; - int total = 0; - int loop = 0; - unsigned long excess; - unsigned long nr_scanned; - struct mem_cgroup_reclaim_cookie reclaim = { - .pgdat = pgdat, - }; - - excess = soft_limit_excess(root_memcg); - - while (1) { - victim = mem_cgroup_iter(root_memcg, victim, &reclaim); - if (!victim) { - loop++; - if (loop >= 2) { - /* - * If we have not been able to reclaim - * anything, it might because there are - * no reclaimable pages under this hierarchy - */ - if (!total) - break; - /* - * We want to do more targeted reclaim. - * excess >> 2 is not to excessive so as to - * reclaim too much, nor too less that we keep - * coming back to reclaim from this cgroup - */ - if (total >= (excess >> 2) || - (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) - break; - } - continue; - } - total += mem_cgroup_shrink_node(victim, gfp_mask, false, - pgdat, &nr_scanned); - *total_scanned += nr_scanned; - if (!soft_limit_excess(root_memcg)) - break; - } - mem_cgroup_iter_break(root_memcg, victim); - return total; -} - -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned) -{ - unsigned long nr_reclaimed = 0; - struct mem_cgroup_per_node *mz, *next_mz = NULL; - unsigned long reclaimed; - int loop = 0; - struct mem_cgroup_tree_per_node *mctz; - unsigned long excess; - - if (lru_gen_enabled()) - return 0; - - if (order > 0) - return 0; - - mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id]; - - /* - * Do not even bother to check the largest node if the root - * is empty. Do it lockless to prevent lock bouncing. Races - * are acceptable as soft limit is best effort anyway. - */ - if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root)) - return 0; - - /* - * This loop can run a while, specially if mem_cgroup's continuously - * keep exceeding their soft limit and putting the system under - * pressure - */ - do { - if (next_mz) - mz = next_mz; - else - mz = mem_cgroup_largest_soft_limit_node(mctz); - if (!mz) - break; - - reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, - gfp_mask, total_scanned); - nr_reclaimed += reclaimed; - spin_lock_irq(&mctz->lock); - - /* - * If we failed to reclaim anything from this memory cgroup - * it is time to move on to the next cgroup - */ - next_mz = NULL; - if (!reclaimed) - next_mz = __mem_cgroup_largest_soft_limit_node(mctz); - - excess = soft_limit_excess(mz->memcg); - /* - * One school of thought says that we should not add - * back the node to the tree if reclaim returns 0. - * But our reclaim could return 0, simply because due - * to priority we are exposing a smaller subset of - * memory to reclaim from. Consider this as a longer - * term TODO. - */ - /* If excess == 0, no tree ops */ - __mem_cgroup_insert_exceeded(mz, mctz, excess); - spin_unlock_irq(&mctz->lock); - css_put(&mz->memcg->css); - loop++; - /* - * Could not reclaim anything and there are no more - * mem cgroups to try or we seem to be looping without - * reclaiming anything. - */ - if (!nr_reclaimed && - (next_mz == NULL || - loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) - break; - } while (!nr_reclaimed); - if (next_mz) - css_put(&next_mz->memcg->css); - return nr_reclaimed; -} - static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, struct cftype *cft) { @@ -511,7 +192,7 @@ static void mem_cgroup_threshold(struct mem_cgroup *memcg) } } -/* Cgroup1: threshold notifications & softlimit tree updates */ +/* Cgroup1: threshold notifications */ /* * Per memcg event counter is incremented at every pagein/pageout. With THP, @@ -519,15 +200,9 @@ static void mem_cgroup_threshold(struct mem_cgroup *memcg) * to trigger some periodic events. This is straightforward and better * than using jiffies etc. to handle periodic memcg event. */ -enum mem_cgroup_events_target { - MEM_CGROUP_TARGET_THRESH, - MEM_CGROUP_TARGET_SOFTLIMIT, - MEM_CGROUP_NTARGETS, -}; - struct memcg1_events_percpu { unsigned long nr_page_events; - unsigned long targets[MEM_CGROUP_NTARGETS]; + unsigned long threshold_target; }; static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages) @@ -544,53 +219,29 @@ static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages) } #define THRESHOLDS_EVENTS_TARGET 128 -#define SOFTLIMIT_EVENTS_TARGET 1024 -static bool memcg1_event_ratelimit(struct mem_cgroup *memcg, - enum mem_cgroup_events_target target) +static bool memcg1_event_ratelimit(struct mem_cgroup *memcg) { unsigned long val, next; val = __this_cpu_read(memcg->events_percpu->nr_page_events); - next = __this_cpu_read(memcg->events_percpu->targets[target]); + next = __this_cpu_read(memcg->events_percpu->threshold_target); /* from time_after() in jiffies.h */ if ((long)(next - val) < 0) { - switch (target) { - case MEM_CGROUP_TARGET_THRESH: - next = val + THRESHOLDS_EVENTS_TARGET; - break; - case MEM_CGROUP_TARGET_SOFTLIMIT: - next = val + SOFTLIMIT_EVENTS_TARGET; - break; - default: - break; - } - __this_cpu_write(memcg->events_percpu->targets[target], next); + __this_cpu_write(memcg->events_percpu->threshold_target, + val + THRESHOLDS_EVENTS_TARGET); return true; } return false; } -/* - * Check events in order. - * - */ -static void memcg1_check_events(struct mem_cgroup *memcg, int nid) +static void memcg1_check_events(struct mem_cgroup *memcg) { if (IS_ENABLED(CONFIG_PREEMPT_RT)) return; - /* threshold event is triggered in finer grain than soft limit */ - if (unlikely(memcg1_event_ratelimit(memcg, - MEM_CGROUP_TARGET_THRESH))) { - bool do_softlimit; - - do_softlimit = memcg1_event_ratelimit(memcg, - MEM_CGROUP_TARGET_SOFTLIMIT); + if (unlikely(memcg1_event_ratelimit(memcg))) mem_cgroup_threshold(memcg); - if (unlikely(do_softlimit)) - memcg1_update_tree(memcg, nid); - } } void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg) @@ -599,7 +250,7 @@ void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg) local_irq_save(flags); memcg1_charge_statistics(memcg, folio_nr_pages(folio)); - memcg1_check_events(memcg, folio_nid(folio)); + memcg1_check_events(memcg); local_irq_restore(flags); } @@ -672,7 +323,7 @@ void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci) VM_WARN_ON_IRQS_ENABLED(); memcg1_charge_statistics(memcg, -folio_nr_pages(folio)); preempt_enable_nested(); - memcg1_check_events(memcg, folio_nid(folio)); + memcg1_check_events(memcg); rcu_read_unlock(); obj_cgroup_put(objcg); @@ -726,14 +377,14 @@ void memcg1_swapin(struct folio *folio) #endif void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, - unsigned long nr_memory, int nid) + unsigned long nr_memory) { unsigned long flags; local_irq_save(flags); count_memcg_events(memcg, PGPGOUT, pgpgout); __this_cpu_add(memcg->events_percpu->nr_page_events, nr_memory); - memcg1_check_events(memcg, nid); + memcg1_check_events(memcg); local_irq_restore(flags); } @@ -2566,22 +2217,3 @@ void memcg1_free_events(struct mem_cgroup *memcg) { free_percpu(memcg->events_percpu); } - -static int __init memcg1_init(void) -{ - int node; - - for_each_node(node) { - struct mem_cgroup_tree_per_node *rtpn; - - rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, node); - - rtpn->rb_root = RB_ROOT; - rtpn->rb_rightmost = NULL; - spin_lock_init(&rtpn->lock); - soft_limit_tree.rb_tree_per_node[node] = rtpn; - } - - return 0; -} -subsys_initcall(memcg1_init); diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index 1e394269c613..0952b2a783e5 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -25,6 +25,9 @@ int memory_stat_show(struct seq_file *m, void *v); struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, unsigned int n); +void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg, + struct mem_cgroup *parent, int idx); + /* Cgroup v1-specific declarations */ #ifdef CONFIG_MEMCG_V1 @@ -34,19 +37,13 @@ static inline bool do_memsw_account(void) return !cgroup_subsys_on_dfl(memory_cgrp_subsys); } -unsigned long memcg_events_local(struct mem_cgroup *memcg, int event); -unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx); -unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item); +unsigned long memcg_events_local(const struct mem_cgroup *memcg, int event); +unsigned long memcg_page_state_local(const struct mem_cgroup *memcg, int idx); +unsigned long memcg_page_state_local_output(const struct mem_cgroup *memcg, int item); bool memcg1_alloc_events(struct mem_cgroup *memcg); void memcg1_free_events(struct mem_cgroup *memcg); void memcg1_memcg_init(struct mem_cgroup *memcg); -void memcg1_remove_from_trees(struct mem_cgroup *memcg); - -static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) -{ - WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX); -} struct cgroup_taskset; void memcg1_css_offline(struct mem_cgroup *memcg); @@ -65,7 +62,7 @@ void memcg1_oom_recover(struct mem_cgroup *memcg); void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg); void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, - unsigned long nr_memory, int nid); + unsigned long nr_memory); void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s); void reparent_memcg1_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent); @@ -73,8 +70,6 @@ void reparent_memcg1_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgr void reparent_memcg_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent, int idx); -void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg, - struct mem_cgroup *parent, int idx); void memcg1_account_kmem(struct mem_cgroup *memcg, int nr_pages); static inline bool memcg1_tcpmem_active(struct mem_cgroup *memcg) @@ -98,8 +93,6 @@ static inline bool memcg1_alloc_events(struct mem_cgroup *memcg) { return true; static inline void memcg1_free_events(struct mem_cgroup *memcg) {} static inline void memcg1_memcg_init(struct mem_cgroup *memcg) {} -static inline void memcg1_remove_from_trees(struct mem_cgroup *memcg) {} -static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) {} static inline void memcg1_css_offline(struct mem_cgroup *memcg) {} static inline bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked) @@ -115,7 +108,7 @@ static inline void memcg1_commit_charge(struct folio *folio, static inline void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, - unsigned long nr_memory, int nid) {} + unsigned long nr_memory) {} static inline void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) {} diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 856a7d07586c..4d00748c8a5b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -62,6 +62,7 @@ #include #include #include +#include #include "internal.h" #include "swap.h" #include "swap_table.h" @@ -232,14 +233,29 @@ static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memc return objcg; } -#ifdef CONFIG_MEMCG_V1 static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force); +/* + * Reparent the non-hierarchical lruvec stats that count_shadow_nodes() reads + * to approximate the shadow node budget. They are not exposed to userspace + * on cgroup v2, but they must follow the reparented folios; otherwise the + * ancestor would only receive the negative deltas when the folios are freed + * without ever having received the positive base, and its local stats would + * permanently underflow. + */ +static void reparent_v2_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) +{ + int i; + + for (i = 0; i < NR_LRU_LISTS; i++) + reparent_memcg_lruvec_state_local(memcg, parent, NR_LRU_BASE + i); + + reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_RECLAIMABLE_B); + reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_UNRECLAIMABLE_B); +} + static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) { - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) - return; - /* * Reparent stats exposed non-hierarchically. Flush @memcg's stats first * to read its stats accurately , and conservatively flush @parent's @@ -248,17 +264,18 @@ static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgr */ __mem_cgroup_flush_stats(memcg, true); - /* The following counts are all non-hierarchical and need to be reparented. */ - reparent_memcg1_state_local(memcg, parent); - reparent_memcg1_lruvec_state_local(memcg, parent); + if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { + reparent_v2_lruvec_state_local(memcg, parent); + } else { +#ifdef CONFIG_MEMCG_V1 + /* The following counts are all non-hierarchical and need to be reparented. */ + reparent_memcg1_state_local(memcg, parent); + reparent_memcg1_lruvec_state_local(memcg, parent); +#endif + } __mem_cgroup_flush_stats(parent, true); } -#else -static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) -{ -} -#endif static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid) { @@ -331,7 +348,7 @@ EXPORT_SYMBOL(memcg_bpf_enabled_key); * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup * is returned. */ -struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio) +struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio) { struct mem_cgroup *memcg; @@ -356,7 +373,7 @@ struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio) * after page_cgroup_ino() returns, so it only should be used by callers that * do not care (such as procfs interfaces). */ -ino_t page_cgroup_ino(struct page *page) +ino_t page_cgroup_ino(const struct page *page) { struct mem_cgroup *memcg; unsigned long ino = 0; @@ -489,9 +506,9 @@ struct lruvec_stats { long state_pending[NR_MEMCG_NODE_STAT_ITEMS]; }; -unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) +unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx) { - struct mem_cgroup_per_node *pn; + const struct mem_cgroup_per_node *pn; long x; int i; @@ -502,7 +519,7 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0; - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); x = READ_ONCE(pn->lruvec_stats->state[i]); #ifdef CONFIG_SMP if (x < 0) @@ -530,10 +547,10 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) * monotonically-incremented event counters are stored in * enum node_stat_item. */ -unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx) { - struct mem_cgroup_per_node *pn; + const struct mem_cgroup_per_node *pn; int i; if (mem_cgroup_disabled()) @@ -543,14 +560,14 @@ unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0; - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); return (unsigned long)READ_ONCE(pn->lruvec_stats->state[i]); } -unsigned long lruvec_page_state_local(struct lruvec *lruvec, +unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx) { - struct mem_cgroup_per_node *pn; + const struct mem_cgroup_per_node *pn; long x; int i; @@ -561,7 +578,7 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec, if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0; - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); x = READ_ONCE(pn->lruvec_stats->state_local[i]); #ifdef CONFIG_SMP if (x < 0) @@ -570,7 +587,6 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec, return x; } -#ifdef CONFIG_MEMCG_V1 static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn, enum node_stat_item idx, long val); @@ -592,7 +608,6 @@ void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg, __mod_memcg_lruvec_state(parent_pn, idx, value); } } -#endif /* Subset of vm_event_item to report for memcg event stats */ static const unsigned int memcg_vm_event_stat[] = { @@ -675,9 +690,11 @@ struct memcg_vmstats { long state[MEMCG_VMSTAT_SIZE]; unsigned long events[NR_MEMCG_EVENTS]; +#ifdef CONFIG_MEMCG_V1 /* Non-hierarchical (CPU aggregated) page state & events */ long state_local[MEMCG_VMSTAT_SIZE]; unsigned long events_local[NR_MEMCG_EVENTS]; +#endif /* Pending child counts during tree propagation */ long state_pending[MEMCG_VMSTAT_SIZE]; @@ -687,6 +704,31 @@ struct memcg_vmstats { atomic_long_t stats_updates; }; +/* + * The non-hierarchical memcg-wide counters are read back only by the legacy + * memory.stat and memory.numa_stat, and by reparenting on offline, all of which + * are v1-only. If the kernel is built without CONFIG_MEMCG_V1, or if the boot + * param cgroup_no_v1= has blocked the memory controller from v1 hierarchies, + * then nothing reads them and writers can skip the updates. + */ +static long *memcg_state_local_array(struct mem_cgroup *memcg) +{ +#ifdef CONFIG_MEMCG_V1 + if (!cgroup1_ssid_disabled(memory_cgrp_id)) + return memcg->vmstats->state_local; +#endif + return NULL; +} + +static unsigned long *memcg_events_local_array(struct mem_cgroup *memcg) +{ +#ifdef CONFIG_MEMCG_V1 + if (!cgroup1_ssid_disabled(memory_cgrp_id)) + return memcg->vmstats->events_local; +#endif + return NULL; +} + /* * memcg and lruvec stats flushing * @@ -757,7 +799,7 @@ static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force) return; if (mem_cgroup_is_root(memcg)) - WRITE_ONCE(flush_last_time, jiffies_64); + WRITE_ONCE(flush_last_time, get_jiffies_64()); css_rstat_flush(&memcg->css); } @@ -785,7 +827,7 @@ void mem_cgroup_flush_stats(struct mem_cgroup *memcg) void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg) { /* Only flush if the periodic flusher is one full cycle late */ - if (time_after64(jiffies_64, READ_ONCE(flush_last_time) + 2*FLUSH_TIME)) + if (time_after64(get_jiffies_64(), READ_ONCE(flush_last_time) + 2 * FLUSH_TIME)) mem_cgroup_flush_stats(memcg); } @@ -799,7 +841,7 @@ static void flush_memcg_stats_dwork(struct work_struct *w) queue_delayed_work(system_dfl_wq, &stats_flush_dwork, FLUSH_TIME); } -unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) +unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx) { long x; int i = memcg_stats_index(idx); @@ -845,16 +887,21 @@ static long memcg_state_val_in_pages(int idx, long val) return val < 0 ? -res : res; } -#ifdef CONFIG_MEMCG_V1 /* - * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race with - * reparenting of non-hierarchical state_locals. + * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race + * with reparenting of non-hierarchical state_locals. Offlining a + * memcg is rare, so do the redirection for all cgroup hierarchies. */ -static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg, - bool *rcu_locked) +static inline struct mem_cgroup * +get_non_dying_memcg_start(struct mem_cgroup *memcg, bool *rcu_locked) { - /* Rebinding can cause this value to be changed at runtime */ - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { + /* + * Fast path: the caller holds a reference to @memcg, so reading + * its CSS_DYING flag without the RCU lock is safe. The RCU lock + * is only needed to walk up to a non-dying ancestor, which + * happens only while a memcg is actually being offlined. + */ + if (!memcg_is_dying(memcg)) { *rcu_locked = false; return memcg; } @@ -875,17 +922,6 @@ static inline void get_non_dying_memcg_end(bool rcu_locked) rcu_read_unlock(); } -#else -static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg, - bool *rcu_locked) -{ - return memcg; -} - -static inline void get_non_dying_memcg_end(bool rcu_locked) -{ -} -#endif static void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, long val) @@ -928,7 +964,7 @@ void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, #ifdef CONFIG_MEMCG_V1 /* idx can be of type enum memcg_stat_item or node_stat_item. */ -unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) +unsigned long memcg_page_state_local(const struct mem_cgroup *memcg, int idx) { long x; int i = memcg_stats_index(idx); @@ -1091,7 +1127,7 @@ void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, put_cpu(); } -unsigned long memcg_events(struct mem_cgroup *memcg, int event) +unsigned long memcg_events(const struct mem_cgroup *memcg, int event) { int i = memcg_events_index(event); @@ -1110,7 +1146,7 @@ bool memcg_vm_event_item_valid(enum vm_event_item idx) } #ifdef CONFIG_MEMCG_V1 -unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) +unsigned long memcg_events_local(const struct mem_cgroup *memcg, int event) { int i = memcg_events_index(event); @@ -1220,7 +1256,7 @@ again: * * See folio_memcg() for folio->objcg/memcg binding rules. */ -struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio) +struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio) { struct mem_cgroup *memcg; @@ -1441,7 +1477,7 @@ void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, * * Return: The lruvec this folio is on with its lock held and rcu read lock held. */ -struct lruvec *folio_lruvec_lock(struct folio *folio) +struct lruvec *folio_lruvec_lock(const struct folio *folio) { struct lruvec *lruvec; @@ -1469,7 +1505,7 @@ retry: * Return: The lruvec this folio is on with its lock held and interrupts * disabled and rcu read lock held. */ -struct lruvec *folio_lruvec_lock_irq(struct folio *folio) +struct lruvec *folio_lruvec_lock_irq(const struct folio *folio) { struct lruvec *lruvec; @@ -1498,7 +1534,7 @@ retry: * Return: The lruvec this folio is on with its lock held and interrupts * disabled and rcu read lock held. */ -struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flags) { struct lruvec *lruvec; @@ -1529,28 +1565,12 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages) { struct mem_cgroup_per_node *mz; - unsigned long *lru_size; - long size; if (mem_cgroup_disabled()) return; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - lru_size = &mz->lru_zone_size[zid][lru]; - - if (nr_pages < 0) - *lru_size += nr_pages; - - size = *lru_size; - if (WARN_ONCE(size < 0, - "%s(%p, %d, %ld): lru_size %ld\n", - __func__, lruvec, lru, nr_pages, size)) { - VM_BUG_ON(1); - *lru_size = 0; - } - - if (nr_pages > 0) - *lru_size += nr_pages; + mz->lru_zone_size[zid][lru] += nr_pages; } /** @@ -1709,14 +1729,14 @@ static int memcg_page_state_output_unit(int item) } } -unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item) +unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item) { return memcg_page_state(memcg, item) * memcg_page_state_output_unit(item); } #ifdef CONFIG_MEMCG_V1 -unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item) +unsigned long memcg_page_state_local_output(const struct mem_cgroup *memcg, int item) { return memcg_page_state_local(memcg, item) * memcg_page_state_output_unit(item); @@ -1812,7 +1832,8 @@ static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is * enabled */ -void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) +void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, + struct task_struct *p) { rcu_read_lock(); @@ -1877,7 +1898,7 @@ void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) /* * Return the memory (and swap, if configured) limit for a memcg. */ -unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) +unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg) { unsigned long max = READ_ONCE(memcg->memory.max); @@ -2048,6 +2069,15 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) * nr_pages in a single cacheline. This may change in future. */ #define NR_MEMCG_STOCK 7 + +/* + * Watermarks for a charge stock slot, in the spirit of pcp->high and + * pcp->batch: MEMCG_STOCK_HIGH is the high watermark at which a slot is + * trimmed, and it is trimmed down to MEMCG_STOCK_LOW rather than emptied. + */ +#define MEMCG_STOCK_LOW (MEMCG_CHARGE_BATCH / 2) +#define MEMCG_STOCK_HIGH (MEMCG_CHARGE_BATCH) + #define FLUSHING_CACHED_CHARGE 0 struct memcg_stock_pcp { local_trylock_t lock; @@ -2228,17 +2258,18 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { struct memcg_stock_pcp *stock; struct mem_cgroup *cached; - uint8_t stock_pages; + unsigned int stock_pages; bool success = false; int empty_slot = -1; int i; /* - * For now limit MEMCG_CHARGE_BATCH to 127 and less. In future if we - * decide to increase it more than 127 then we will need more careful - * handling of nr_pages[] in struct memcg_stock_pcp. + * nr_pages[] is a uint8_t and a slot's count is capped at + * MEMCG_STOCK_HIGH. Raising MEMCG_CHARGE_BATCH beyond 127 would need + * more careful handling of nr_pages[] in struct memcg_stock_pcp. */ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S8_MAX); + BUILD_BUG_ON(MEMCG_STOCK_HIGH > U8_MAX); VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg)); @@ -2259,9 +2290,12 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) empty_slot = i; if (memcg == READ_ONCE(stock->cached[i])) { stock_pages = READ_ONCE(stock->nr_pages[i]) + nr_pages; + if (stock_pages > MEMCG_STOCK_HIGH) { + memcg_uncharge(memcg, + stock_pages - MEMCG_STOCK_LOW); + stock_pages = MEMCG_STOCK_LOW; + } WRITE_ONCE(stock->nr_pages[i], stock_pages); - if (stock_pages > MEMCG_CHARGE_BATCH) - drain_stock(stock, i); success = true; break; } @@ -2306,7 +2340,7 @@ static bool is_memcg_drain_needed(struct memcg_stock_pcp *stock, return flush; } -static void schedule_drain_work(int cpu, struct work_struct *work) +static bool schedule_drain_work(int cpu, struct work_struct *work) { /* * Protect housekeeping cpumask read and work enqueue together @@ -2315,8 +2349,11 @@ static void schedule_drain_work(int cpu, struct work_struct *work) * pending work on newly isolated CPUs. */ guard(rcu)(); - if (!cpu_is_isolated(cpu)) - queue_work_on(cpu, memcg_wq, work); + if (cpu_is_isolated(cpu)) + return false; + + queue_work_on(cpu, memcg_wq, work); + return true; } /* @@ -2348,8 +2385,9 @@ void drain_all_stock(struct mem_cgroup *root_memcg) &memcg_st->flags)) { if (cpu == curcpu) drain_local_memcg_stock(&memcg_st->work); - else - schedule_drain_work(cpu, &memcg_st->work); + else if (!schedule_drain_work(cpu, &memcg_st->work)) + clear_bit(FLUSHING_CACHED_CHARGE, + &memcg_st->flags); } if (!test_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags) && @@ -2358,8 +2396,9 @@ void drain_all_stock(struct mem_cgroup *root_memcg) &obj_st->flags)) { if (cpu == curcpu) drain_local_obj_stock(&obj_st->work); - else - schedule_drain_work(cpu, &obj_st->work); + else if (!schedule_drain_work(cpu, &obj_st->work)) + clear_bit(FLUSHING_CACHED_CHARGE, + &obj_st->flags); } } migrate_enable(); @@ -2368,9 +2407,21 @@ void drain_all_stock(struct mem_cgroup *root_memcg) static int memcg_hotplug_cpu_dead(unsigned int cpu) { + struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu); + struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu); + /* no need for the local lock */ - drain_obj_stock(&per_cpu(obj_stock, cpu)); - drain_stock_fully(&per_cpu(memcg_stock, cpu)); + drain_obj_stock(obj_st); + drain_stock_fully(memcg_st); + + /* + * A drain work queued before the CPU went away is executed by an + * unbound worker on some other CPU and clears that CPU's flag, so + * clear the flags here to make these stocks drainable again once + * the CPU comes back online. + */ + clear_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags); + clear_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags); return 0; } @@ -2410,6 +2461,11 @@ static void high_work_func(struct work_struct *work) reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); } +static void high_irq_work_func(struct irq_work *work) +{ + schedule_work(&container_of(work, struct mem_cgroup, high_irq_work)->high_work); +} + /* * Clamp the maximum sleep time per allocation batch to 2 seconds. This is * enough to still cause a significant slowdown in most cases, while still @@ -2515,8 +2571,7 @@ static u64 swap_find_max_overage(struct mem_cgroup *memcg) * Get the number of jiffies that we should penalise a mischievous cgroup which * is exceeding its memory.high by checking both it and its ancestors. */ -static unsigned long calculate_high_delay(struct mem_cgroup *memcg, - unsigned int nr_pages, +static unsigned long calculate_high_delay(unsigned int nr_pages, u64 max_overage) { unsigned long penalty_jiffies; @@ -2594,10 +2649,10 @@ retry_reclaim: * memory.high is breached and reclaim is unable to keep up. Throttle * allocators proactively to slow down excessive growth. */ - penalty_jiffies = calculate_high_delay(memcg, nr_pages, + penalty_jiffies = calculate_high_delay(nr_pages, mem_find_max_overage(memcg)); - penalty_jiffies += calculate_high_delay(memcg, nr_pages, + penalty_jiffies += calculate_high_delay(nr_pages, swap_find_max_overage(memcg)); /* @@ -2656,10 +2711,11 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, bool raised_max_event = false; unsigned long pflags; bool allow_spinning = gfpflags_allow_spinning(gfp_mask); + int ret = 0; retry: if (consume_stock(memcg, nr_pages)) - return 0; + return ret; if (!allow_spinning) /* Avoid the refill and flush of the older stock */ @@ -2770,16 +2826,11 @@ nomem: * put the burden of reclaim on regular allocation requests * and let these go through as privileged allocations. */ - if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) - return -ENOMEM; + if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) { + ret = -ENOMEM; + goto out; + } force: - /* - * If the allocation has to be enforced, don't forget to raise - * a MEMCG_MAX event. - */ - if (!raised_max_event) - __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); - /* * The allocation either can't fail or will lead to more memory * being freed very soon. Allow memory usage go over the limit @@ -2789,7 +2840,15 @@ force: if (do_memsw_account()) page_counter_charge(&memcg->memsw, nr_pages); - return 0; +out: + /* + * Don't forget to raise a MEMCG_MAX event for forced or rejected + * requests. + */ + if (!raised_max_event) + __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); + + return ret; done_restock: if (batch > nr_pages) @@ -2815,7 +2874,10 @@ done_restock: /* Don't bother a random interrupted task */ if (!in_task()) { if (mem_high) { - schedule_work(&memcg->high_work); + if (allow_spinning) + schedule_work(&memcg->high_work); + else + irq_work_queue(&memcg->high_irq_work); break; } continue; @@ -2848,7 +2910,7 @@ done_restock: !(current->flags & PF_MEMALLOC) && gfpflags_allow_blocking(gfp_mask)) __mem_cgroup_handle_over_high(gfp_mask); - return 0; + return ret; } static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, @@ -3093,7 +3155,7 @@ from_memcg: return rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1); } -struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio) +struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio) { struct obj_cgroup *objcg; @@ -3756,7 +3818,7 @@ static void memcg_online_kmem(struct mem_cgroup *memcg) static_branch_enable(&memcg_kmem_online_key); - memcg->kmemcg_id = memcg->id.id; + memcg->kmemcg_id = memcg->private_id; } static void memcg_offline_kmem(struct mem_cgroup *memcg) @@ -3946,7 +4008,7 @@ void mem_cgroup_flush_foreign(struct bdi_writeback *wb) { struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10); - u64 now = jiffies_64; + u64 now = get_jiffies_64(); int i; for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { @@ -4015,15 +4077,15 @@ static DEFINE_XARRAY_ALLOC1(mem_cgroup_private_ids); static void mem_cgroup_private_id_remove(struct mem_cgroup *memcg) { - if (memcg->id.id > 0) { - xa_erase(&mem_cgroup_private_ids, memcg->id.id); - memcg->id.id = 0; + if (memcg->private_id > 0) { + xa_erase(&mem_cgroup_private_ids, memcg->private_id); + memcg->private_id = 0; } } static inline void mem_cgroup_private_id_put(struct mem_cgroup *memcg, unsigned int n) { - if (refcount_sub_and_test(n, &memcg->id.ref)) { + if (refcount_sub_and_test(n, &memcg->private_id_ref)) { mem_cgroup_private_id_remove(memcg); /* Memcg ID pins CSS */ @@ -4033,7 +4095,7 @@ static inline void mem_cgroup_private_id_put(struct mem_cgroup *memcg, unsigned struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, unsigned int n) { - while (!refcount_add_not_zero(n, &memcg->id.ref)) { + while (!refcount_add_not_zero(n, &memcg->private_id_ref)) { /* * The root cgroup cannot be destroyed, so it's refcount must * always be >= 1. @@ -4157,7 +4219,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) if (!memcg) return ERR_PTR(-ENOMEM); - error = xa_alloc(&mem_cgroup_private_ids, &memcg->id.id, NULL, + error = xa_alloc(&mem_cgroup_private_ids, &memcg->private_id, NULL, XA_LIMIT(1, MEM_CGROUP_ID_MAX), GFP_KERNEL); if (error) goto fail; @@ -4190,6 +4252,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) goto fail; INIT_WORK(&memcg->high_work, high_work_func); + init_irq_work(&memcg->high_irq_work, high_irq_work_func); vmpressure_init(&memcg->vmpressure); INIT_LIST_HEAD(&memcg->memory_peaks); INIT_LIST_HEAD(&memcg->swap_peaks); @@ -4228,7 +4291,6 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) return ERR_CAST(memcg); page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); - memcg1_soft_limit_reset(memcg); #ifdef CONFIG_ZSWAP memcg->zswap_max = PAGE_COUNTER_MAX; WRITE_ONCE(memcg->zswap_writeback, true); @@ -4304,7 +4366,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) lru_gen_online_memcg(memcg); /* Online state pins memcg ID, memcg ID pins CSS */ - refcount_set(&memcg->id.ref, 1); + refcount_set(&memcg->private_id_ref, 1); css_get(css); /* @@ -4317,7 +4379,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) * publish it here at the end of onlining. This matches the * regular ID destruction during offlining. */ - xa_store(&mem_cgroup_private_ids, memcg->id.id, memcg, GFP_KERNEL); + xa_store(&mem_cgroup_private_ids, memcg->private_id, memcg, GFP_KERNEL); return 0; free_objcg: @@ -4399,8 +4461,8 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css) static_branch_dec(&memcg_bpf_enabled_key); vmpressure_cleanup(&memcg->vmpressure); + irq_work_sync(&memcg->high_irq_work); cancel_work_sync(&memcg->high_work); - memcg1_remove_from_trees(memcg); free_shrinker_info(memcg); mem_cgroup_free(memcg); } @@ -4436,7 +4498,6 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_min(&memcg->memory, 0); page_counter_set_low(&memcg->memory, 0); page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); - memcg1_soft_limit_reset(memcg); page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); memcg_wb_domain_size_changed(memcg); } @@ -4444,7 +4505,10 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) struct aggregate_control { /* pointer to the aggregated (CPU and subtree aggregated) counters */ long *aggregate; - /* pointer to the non-hierarchichal (CPU aggregated) counters */ + /* + * pointer to the non-hierarchical (CPU aggregated) counters or NULL to + * skip updating them (see memcg_state_local_array()) + */ long *local; /* pointer to the pending child counters during tree propagation */ long *pending; @@ -4483,7 +4547,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac) } /* Aggregate counts on this level and propagate upwards */ - if (delta_cpu) + if (delta_cpu && ac->local) ac->local[i] += delta_cpu; if (delta) { @@ -4497,6 +4561,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac) #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) { + long *state_local = memcg_state_local_array(memcg); int nid; if (atomic_read(&memcg->kmem_stat)) { @@ -4504,7 +4569,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) int index = memcg_stats_index(MEMCG_KMEM); memcg->vmstats->state[index] += kmem; - memcg->vmstats->state_local[index] += kmem; + if (state_local) + state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; } @@ -4526,7 +4592,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; - memcg->vmstats->state_local[index] += slab; + if (state_local) + state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4539,7 +4606,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; - memcg->vmstats->state_local[index] += slab; + if (state_local) + state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4564,7 +4632,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) ac = (struct aggregate_control) { .aggregate = memcg->vmstats->state, - .local = memcg->vmstats->state_local, + .local = memcg_state_local_array(memcg), .pending = memcg->vmstats->state_pending, .ppending = parent ? parent->vmstats->state_pending : NULL, .cstat = statc->state, @@ -4575,7 +4643,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) ac = (struct aggregate_control) { .aggregate = memcg->vmstats->events, - .local = memcg->vmstats->events_local, + .local = memcg_events_local_array(memcg), .pending = memcg->vmstats->events_pending, .ppending = parent ? parent->vmstats->events_pending : NULL, .cstat = statc->events, @@ -4719,6 +4787,7 @@ static int memory_peak_show(struct seq_file *sf, void *v) { struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); + guard(spinlock)(&memcg->peaks_lock); return peak_show(sf, v, &memcg->memory); } @@ -4748,7 +4817,7 @@ static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off, struct page_counter *pc, struct list_head *watchers) { - unsigned long usage; + unsigned long usage, old_watermark; struct cgroup_of_peak *peer_ctx; struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); struct cgroup_of_peak *ofp = of_peak(of); @@ -4756,11 +4825,12 @@ static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes, spin_lock(&memcg->peaks_lock); usage = page_counter_read(pc); + old_watermark = READ_ONCE(pc->local_watermark); WRITE_ONCE(pc->local_watermark, usage); list_for_each_entry(peer_ctx, watchers, list) - if (usage > peer_ctx->value) - WRITE_ONCE(peer_ctx->value, usage); + if (peer_ctx != ofp && old_watermark > peer_ctx->value) + WRITE_ONCE(peer_ctx->value, old_watermark); /* initial write, register watcher */ if (ofp->value == OFP_PEAK_UNSET) @@ -5198,6 +5268,51 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, page_counter_calculate_protection(&root->memory, &memcg->memory, recursive_protection); } +#ifdef CONFIG_LRU_GEN +/** + * mem_cgroup_calculate_protection_path - compute protection along a path + * @root: the top ancestor of the sub-tree being checked (NULL for root_mem_cgroup) + * @memcg: the target memory cgroup + * + * Walk the ancestor path from @root down to @memcg and compute the effective + * protection at each level. This is safe for isolated queries because it + * ensures parents are computed before children. + */ +void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg) +{ + bool recursive_protection = + cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT; + struct cgroup *cg; + int root_level, i; + + if (mem_cgroup_disabled()) + return; + + if (!root) + root = root_mem_cgroup; + + if (memcg == root) + return; + + root_level = root->css.cgroup->level; + cg = memcg->css.cgroup; + + rcu_read_lock(); + for (i = root_level + 1; i <= cg->level; i++) { + struct mem_cgroup *cur; + + cur = mem_cgroup_from_css(cgroup_css(cg->ancestors[i], + &memory_cgrp_subsys)); + if (cur) + page_counter_calculate_protection(&root->memory, + &cur->memory, + recursive_protection); + } + rcu_read_unlock(); +} +#endif /* CONFIG_LRU_GEN */ + static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, gfp_t gfp) { @@ -5233,6 +5348,7 @@ int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) /** * mem_cgroup_charge_hugetlb - charge the memcg for a hugetlb folio * @folio: folio being charged + * @mm: mm context of the allocation target * @gfp: reclaim mode * * This function is called when allocating a huge page folio, after the page has @@ -5242,9 +5358,10 @@ int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) * Returns ENOMEM if the memcg is already full. * Returns 0 if either the charge was successful, or if we skip the charging. */ -int mem_cgroup_charge_hugetlb(struct folio *folio, gfp_t gfp) +int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, + gfp_t gfp) { - struct mem_cgroup *memcg = get_mem_cgroup_from_current(); + struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); int ret = 0; /* @@ -5302,7 +5419,6 @@ struct uncharge_gather { unsigned long nr_memory; unsigned long pgpgout; unsigned long nr_kmem; - int nid; }; static inline void uncharge_gather_clear(struct uncharge_gather *ug) @@ -5325,7 +5441,7 @@ static void uncharge_batch(const struct uncharge_gather *ug) memcg1_oom_recover(memcg); } - memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory, ug->nid); + memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory); rcu_read_unlock(); /* drop reference from uncharge_folio */ @@ -5354,7 +5470,6 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug) uncharge_gather_clear(ug); } ug->objcg = objcg; - ug->nid = folio_nid(folio); /* pairs with obj_cgroup_put in uncharge_batch */ obj_cgroup_get(objcg); @@ -5684,6 +5799,108 @@ __setup("cgroup.memory=", cgroup_memory); * basically everything that doesn't depend on a specific mem_cgroup structure * should be initialized from here. */ +/* + * Fields are grouped by access pattern. Putting a field in the wrong group + * breaks the build here. + */ +static void __init memcg_struct_check(void) +{ + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + socket_pressure); +#if BITS_PER_LONG < 64 + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + socket_pressure_seqlock); +#endif + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + memory_events); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + memory_events_local); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + vmpressure); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + private_id_ref); +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + kmem_stat); +#endif + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + high_irq_work); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + high_work); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + memory_peaks); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + swap_peaks); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + peaks_lock); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + events_file); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + events_local_file); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + swap_events_file); +#ifdef CONFIG_CGROUP_WRITEBACK + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + cgwb_list); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + cgwb_domain); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + cgwb_frn); +#endif +#ifdef CONFIG_LRU_GEN_WALKS_MMU + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + mm_list); +#endif + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + vmstats_percpu); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + vmstats); +#ifdef CONFIG_ZSWAP + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + zswap_max); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + zswap_writeback); +#endif + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + private_id); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + kmemcg_id); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + oom_group); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, memcg); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, lruvec_stats_percpu); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, lruvec_stats); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, shrinker_info); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, objcg); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_lruvec, lruvec); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, lru_zone_size); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, iter); +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, slab_reclaimable); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, slab_unreclaimable); +#endif + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_cold, orig_objcg); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_cold, objcg_list); +} + int __init mem_cgroup_init(void) { unsigned int memcg_size; @@ -5697,6 +5914,8 @@ int __init mem_cgroup_init(void) */ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE); + memcg_struct_check(); + cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, memcg_hotplug_cpu_dead); @@ -5798,20 +6017,40 @@ void __mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages) rcu_read_unlock(); } -long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) +long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg) { long nr_swap_pages = get_nr_swap_pages(); - if (mem_cgroup_disabled() || do_memsw_account()) - return nr_swap_pages; - for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) - nr_swap_pages = min_t(long, nr_swap_pages, - READ_ONCE(memcg->swap.max) - - page_counter_read(&memcg->swap)); + if (!mem_cgroup_disabled() && !do_memsw_account()) + nr_swap_pages = min(nr_swap_pages, page_counter_margin(&memcg->swap)); + return nr_swap_pages; } -bool mem_cgroup_swap_full(struct folio *folio) +/** + * mem_cgroup_get_folio_swap_margin - get a folio's memcg swap margin + * @folio: folio whose memcg margin is queried + * + * Return: Remaining chargeable pages in the folio's memcg hierarchy. + */ +long mem_cgroup_get_folio_swap_margin(const struct folio *folio) +{ + struct mem_cgroup *memcg; + long margin; + + if (mem_cgroup_disabled() || do_memsw_account() || + !folio_memcg_charged(folio)) + return PAGE_COUNTER_MAX; + + rcu_read_lock(); + memcg = folio_memcg(folio); + margin = page_counter_margin(&memcg->swap); + rcu_read_unlock(); + + return margin; +} + +bool mem_cgroup_swap_full(const struct folio *folio) { struct mem_cgroup *memcg; bool ret = false; @@ -5864,6 +6103,7 @@ static int swap_peak_show(struct seq_file *sf, void *v) { struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); + guard(spinlock)(&memcg->peaks_lock); return peak_show(sf, v, &memcg->swap); } @@ -6078,7 +6318,7 @@ void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) rcu_read_unlock(); } -bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) +bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg) { /* if zswap is disabled, do not block pages going to the swapping device */ if (!zswap_is_enabled()) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index a8b03e2920ba..d237f556b3a0 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -847,24 +847,25 @@ static int kill_accessing_process(struct task_struct *p, unsigned long pfn, } /* - * MF_IGNORED - The m-f() handler marks the page as PG_hwpoisoned'ed. + * MF_IGNORED - The m-f() handler marks the page as PG_hwpoison'ed. * But it could not do more to isolate the page from being accessed again, * nor does it kill the process. This is extremely rare and one of the * potential causes is that the page state has been changed due to * underlying race condition. This is the most severe outcomes. * - * MF_FAILED - The m-f() handler marks the page as PG_hwpoisoned'ed. + * MF_FAILED - The m-f() handler marks the page as PG_hwpoison'ed. * It should have killed the process, but it can't isolate the page, * due to conditions such as extra pin, unmap failure, etc. Accessing * the page again may trigger another MCE and the process will be killed * by the m-f() handler immediately. * - * MF_DELAYED - The m-f() handler marks the page as PG_hwpoisoned'ed. - * The page is unmapped, and is removed from the LRU or file mapping. - * An attempt to access the page again will trigger page fault and the - * PF handler will kill the process. + * MF_DELAYED - The m-f() handler marks the page as PG_hwpoison'ed. + * It means the page was unmapped and partially isolated (e.g. removed from + * file mapping or the LRU) but full cleanup is deferred (e.g. the metadata + * for the memory, as in struct page/folio, is still referenced). Any + * further access to the page will result in the process being killed. * - * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoisoned'ed. + * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoison'ed. * The page has been completely isolated, that is, unmapped, taken out of * the buddy system, or hole-punched out of the file mapping. */ @@ -938,10 +939,12 @@ static int truncate_error_folio(struct folio *folio, unsigned long pfn, if (mapping->a_ops->error_remove_folio) { int err = mapping->a_ops->error_remove_folio(mapping, folio); - if (err != 0) + if (err == MF_DELAYED) + ret = err; + else if (err != 0) pr_info("%#lx: Failed to punch page: %d\n", pfn, err); else if (!filemap_release_folio(folio, GFP_NOIO)) - pr_info("%#lx: failed to release buffers\n", pfn); + pr_info("%#lx: Failed to release buffers\n", pfn); else ret = MF_RECOVERED; } else { @@ -1036,18 +1039,14 @@ static int me_pagecache_clean(struct page_state *ps, struct page *p) goto out; } - /* - * The shmem page is kept in page cache instead of truncating - * so is expected to have an extra refcount after error-handling. - */ - extra_pins = shmem_mapping(mapping); - /* * Truncation is a bit tricky. Enable it per file system for now. * * Open: to take i_rwsem or not for this? Right now we don't. */ ret = truncate_error_folio(folio, page_to_pfn(p), mapping); + + extra_pins = ret == MF_DELAYED; if (has_extra_refcount(ps, p, extra_pins)) ret = MF_FAILED; diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 54851d8a195b..25e121851b58 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -370,7 +370,7 @@ int next_demotion_node(int node, const nodemask_t *allowed_mask) * closest demotion target. */ nodes_complement(mask, *allowed_mask); - return find_next_best_node(node, &mask); + return find_next_best_node_in(node, &mask, &node_states[N_MEMORY]); } static void disable_all_demotion_targets(void) @@ -450,7 +450,7 @@ static void establish_demotion_targets(void) memtier = list_next_entry(memtier, list); tier_nodes = get_memtier_nodemask(memtier); /* - * find_next_best_node, use 'used' nodemask as a skip list. + * find_next_best_node_in, use 'used' nodemask as a skip list. * Add all memory nodes except the selected memory tier * nodelist to skip list so that we find the best node from the * memtier nodelist. @@ -463,7 +463,8 @@ static void establish_demotion_targets(void) * in the preferred mask when allocating pages during demotion. */ do { - target = find_next_best_node(node, &tier_nodes); + target = find_next_best_node_in(node, &tier_nodes, + &node_states[N_MEMORY]); if (target == NUMA_NO_NODE) break; diff --git a/mm/memory.c b/mm/memory.c index 8b0c2c735d3d..7d821b110df0 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -492,34 +492,10 @@ static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) add_mm_counter(mm, i, rss[i]); } -static bool is_bad_page_map_ratelimited(void) -{ - static unsigned long resume; - static unsigned long nr_shown; - static unsigned long nr_unshown; +/* Allow a burst of 60 bad page map reports per minute. */ +static DEFINE_RATELIMIT_STATE(bad_page_map_ratelimit, 60 * HZ, 60); - /* - * Allow a burst of 60 reports, then keep quiet for that minute; - * or allow a steady drip of one report per second. - */ - if (nr_shown == 60) { - if (time_before(jiffies, resume)) { - nr_unshown++; - return true; - } - if (nr_unshown) { - pr_alert("BUG: Bad page map: %lu messages suppressed\n", - nr_unshown); - nr_unshown = 0; - } - nr_shown = 0; - } - if (nr_shown++ == 0) - resume = jiffies + 60 * HZ; - return false; -} - -static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) +void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) { if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) { snprintf(buf, buf_size, "overflow"); @@ -546,19 +522,6 @@ static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry } } -#define ptval_to_str(buf, val) \ - do { \ - auto __val = (val); \ - \ - ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ - } while (0) - -#if defined(__SIZEOF_INT128__) -#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ -#else -#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ -#endif - static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr) { char pgd_str[PTVAL_STR_MAX]; @@ -633,7 +596,7 @@ static void print_bad_page_map(struct vm_area_struct *vma, char entry_str[PTVAL_STR_MAX]; pgoff_t index, anon_index; - if (is_bad_page_map_ratelimited()) + if (!__ratelimit(&bad_page_map_ratelimit)) return; mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; @@ -2565,20 +2528,22 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, unsigned long curr_page_idx = 0; unsigned long remaining_pages_total = *num; unsigned long pages_to_write_in_pmd; - int ret; + int err = 0; more: - ret = -EFAULT; pmd = walk_to_pmd(mm, addr); - if (!pmd) + if (!pmd) { + err = -ENOMEM; goto out; + } pages_to_write_in_pmd = min_t(unsigned long, remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); /* Allocate the PTE if necessary; takes PMD lock once only. */ - ret = -ENOMEM; - if (pte_alloc(mm, pmd)) + if (pte_alloc(mm, pmd)) { + err = -ENOMEM; goto out; + } while (pages_to_write_in_pmd) { int pte_idx = 0; @@ -2586,15 +2551,14 @@ more: start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); if (!start_pte) { - ret = -EFAULT; + err = -EFAULT; goto out; } for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { - int err = insert_page_in_batch_locked(vma, pte, - addr, pages[curr_page_idx], prot); + err = insert_page_in_batch_locked(vma, pte, addr, + pages[curr_page_idx], prot); if (unlikely(err)) { pte_unmap_unlock(start_pte, pte_lock); - ret = err; remaining_pages_total -= pte_idx; goto out; } @@ -2607,10 +2571,9 @@ more: } if (remaining_pages_total) goto more; - ret = 0; out: *num = remaining_pages_total; - return ret; + return err; } /** @@ -2897,6 +2860,9 @@ vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); + if (unlikely(is_zero_pfn(pfn))) + return VM_FAULT_SIGBUS; + if (addr < vma->vm_start || addr >= vma->vm_end) return VM_FAULT_SIGBUS; @@ -4926,18 +4892,21 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto unlock; /* - * Get a page reference while we know the page can't be - * freed. + * Get a folio reference while we know the folio can't + * be freed. */ - if (trylock_page(vmf->page)) { + folio = page_folio(vmf->page); + if (folio_trylock(folio)) { struct dev_pagemap *pgmap; - get_page(vmf->page); + folio_get(folio); pte_unmap_unlock(vmf->pte, vmf->ptl); pgmap = page_pgmap(vmf->page); ret = pgmap->ops->migrate_to_ram(vmf); - unlock_page(vmf->page); - put_page(vmf->page); + /* migrate_to_ram() might have split the folio. */ + folio = page_folio(vmf->page); + folio_unlock(folio); + folio_put(folio); } else { pte_unmap(vmf->pte); softleaf_entry_wait_on_locked(entry, vmf->ptl); @@ -4953,10 +4922,13 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out; } - /* Prevent swapoff from happening to us. */ + /* Prevent swapoff from happening to us, and reject a bad entry. */ si = get_swap_device(entry); - if (unlikely(!si)) + if (IS_ERR_OR_NULL(si)) { + if (IS_ERR(si)) + ret = VM_FAULT_SIGBUS; goto out; + } folio = swap_cache_get_folio(entry); if (folio) @@ -5265,7 +5237,7 @@ unlock: if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl); out: - if (si) + if (!IS_ERR_OR_NULL(si)) put_swap_device(si); return ret; out_nomap: @@ -6817,7 +6789,6 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, !vma_is_cow_mapping(vma))) return VM_FAULT_SIGSEGV; } -#ifdef CONFIG_PER_VMA_LOCK /* * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of * the assumption that lock is dropped on VM_FAULT_RETRY. @@ -6826,7 +6797,6 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) == (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT))) return VM_FAULT_SIGSEGV; -#endif return 0; } @@ -7164,6 +7134,12 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr }; + /* + * Limit access to one page at a time, as that's what follow_pfnmap_start() + * guarantees; expect the caller to retry to read larger ranges. + */ + len = min_t(int, len, PAGE_SIZE - offset); + retry: if (follow_pfnmap_start(&args)) return -EINVAL; @@ -7175,7 +7151,7 @@ retry: if ((write & FOLL_WRITE) && !writable) return -EINVAL; - maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot); + maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); if (!maddr) return -ENOMEM; @@ -7183,7 +7159,7 @@ retry: goto out_unmap; if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || - (phys_addr != (args.pfn << PAGE_SHIFT)) || + (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 226ab9cb078a..d7a59167bec4 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -43,6 +43,7 @@ #include "mm_init.h" #include "page_alloc.h" #include "shuffle.h" +#include "sparse.h" enum { MEMMAP_ON_MEMORY_DISABLE = 0, @@ -554,8 +555,9 @@ void remove_pfn_range_from_zone(struct zone *zone, /* Select all remaining pages up to the next section boundary */ cur_nr_pages = min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn); - page_init_poison(pfn_to_page(pfn), - sizeof(struct page) * cur_nr_pages); + if (!section_vmemmap_optimizable(__pfn_to_section(pfn))) + page_init_poison(pfn_to_page(pfn), + sizeof(struct page) * cur_nr_pages); } /* @@ -1785,6 +1787,37 @@ bool mhp_range_allowed(u64 start, u64 size, bool need_mapping) } #ifdef CONFIG_MEMORY_HOTREMOVE + +static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) +{ + int nid = *(int *)arg; + + /* + * If a memory block belongs to multiple nodes, the stored nid is not + * reliable. However, such blocks are always online (e.g., cannot get + * offlined) and, therefore, are still spanned by the node. + */ + return mem->nid == nid ? -EEXIST : 0; +} + +/* Caller must hold the memory hotplug lock for this check. */ +static bool node_is_memoryless(int nid) +{ + /* + * A node still spanning pages (especially ZONE_DEVICE) is not + * memoryless. A node spans memory after move_pfn_range_to_zone(), + * e.g. once a memory block has been onlined. + */ + if (node_spanned_pages(nid)) + return false; + /* + * Offline memory blocks may not be spanned by the node yet, but they + * link to it in sysfs and can be onlined later, so the node is not + * memoryless while any remain. + */ + return !for_each_memory_block(&nid, check_no_memblock_for_node_cb); +} + /* * Scan pfn range [start,end) to find movable/migratable pages (LRU and * hugetlb folio, movable_ops pages). Will skip over most unmovable @@ -2214,18 +2247,6 @@ static int check_cpu_on_node(int nid) return 0; } -static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) -{ - int nid = *(int *)arg; - - /* - * If a memory block belongs to multiple nodes, the stored nid is not - * reliable. However, such blocks are always online (e.g., cannot get - * offlined) and, therefore, are still spanned by the node. - */ - return mem->nid == nid ? -EEXIST : 0; -} - /** * try_offline_node * @nid: the node ID @@ -2237,23 +2258,7 @@ static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) */ void try_offline_node(int nid) { - int rc; - - /* - * If the node still spans pages (especially ZONE_DEVICE), don't - * offline it. A node spans memory after move_pfn_range_to_zone(), - * e.g., after the memory block was onlined. - */ - if (node_spanned_pages(nid)) - return; - - /* - * Especially offline memory blocks might not be spanned by the - * node. They will get spanned by the node once they get onlined. - * However, they link to the node in sysfs and can get onlined later. - */ - rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb); - if (rc) + if (!node_is_memoryless(nid)) return; if (check_cpu_on_node(nid)) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 79053ece02cd..e9860fb9f73f 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -112,6 +112,7 @@ #include #include #include +#include #include #include @@ -157,6 +158,7 @@ static const int weightiness = 32; */ struct weighted_interleave_state { bool mode_auto; + struct rcu_head rcu; u8 iw_table[]; }; static struct weighted_interleave_state __rcu *wi_state; @@ -168,6 +170,24 @@ static unsigned int *node_bw_table; */ static DEFINE_MUTEX(wi_state_lock); +/* Readers that sleep while walking iw_table hold this instead */ +DEFINE_STATIC_SRCU_FAST(wi_srcu); + +static void wi_state_free_rcu(struct rcu_head *head) +{ + struct weighted_interleave_state *state = + container_of(head, struct weighted_interleave_state, rcu); + + kfree_rcu(state, rcu); +} + +/* Retire through both flavors: sleeping readers use SRCU, the rest RCU */ +static void wi_state_retire(struct weighted_interleave_state *state) +{ + if (state) + call_srcu(&wi_srcu, &state->rcu, wi_state_free_rcu); +} + static u8 get_il_weight(int node) { struct weighted_interleave_state *state; @@ -266,10 +286,7 @@ int mempolicy_set_node_perf(unsigned int node, struct access_coordinate *coords) rcu_assign_pointer(wi_state, new_wi_state); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); out: kfree(old_bw); return 0; @@ -650,7 +667,8 @@ static inline bool queue_folio_required(struct folio *folio, return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT); } -static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) +static void queue_folios_pmd(pmd_t *pmd, unsigned long addr, + struct mm_walk *walk) { struct folio *folio; struct queue_pages *qp = walk->private; @@ -661,11 +679,9 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) qp->nr_failed++; return; } - folio = pmd_folio(pmdval); - if (is_huge_zero_folio(folio)) { - walk->action = ACTION_CONTINUE; + folio = vm_normal_folio_pmd(walk->vma, addr, pmdval); + if (!folio || folio_is_zone_device(folio)) return; - } if (!queue_folio_required(folio, qp)) return; if (!(qp->flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) || @@ -698,7 +714,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, ptl = pmd_trans_huge_lock(pmd, vma); if (ptl) { - queue_folios_pmd(pmd, walk); + queue_folios_pmd(pmd, addr, walk); spin_unlock(ptl); goto out; } @@ -2180,34 +2196,15 @@ unsigned int mempolicy_slab_node(void) } } -static unsigned int read_once_policy_nodemask(struct mempolicy *pol, - nodemask_t *mask) -{ - /* - * barrier stabilizes the nodemask locally so that it can be iterated - * over safely without concern for changes. Allocators validate node - * selection does not violate mems_allowed, so this is safe. - */ - barrier(); - memcpy(mask, &pol->nodes, sizeof(nodemask_t)); - barrier(); - return nodes_weight(*mask); -} - static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx) { struct weighted_interleave_state *state; - nodemask_t nodemask; - unsigned int target, nr_nodes; + unsigned int target, nnodes = 0; u8 *table = NULL; unsigned int weight_total = 0; u8 weight; int nid = 0; - nr_nodes = read_once_policy_nodemask(pol, &nodemask); - if (!nr_nodes) - return numa_node_id(); - rcu_read_lock(); state = rcu_dereference(wi_state); @@ -2215,22 +2212,45 @@ static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx) if (state) table = state->iw_table; - /* calculate the total weight */ - for_each_node_mask(nid, nodemask) + /* calculate the total weight and the node count */ + for_each_node_mask(nid, pol->nodes) { weight_total += table ? table[nid] : 1; + nnodes++; + } + + /* the mask is empty */ + if (!weight_total) { + rcu_read_unlock(); + return numa_node_id(); + } /* Calculate the node offset based on totals */ target = ilx % weight_total; - nid = first_node(nodemask); - while (target) { + nid = first_node(pol->nodes); + + /* + * The target was calculated in a separate loop, and a concurrent + * rebind can change the contents of pol->nodes as we calculate. + * Access is safe, in the worst case we suddenly perceive an empty + * nodemask and return numa_node_id() below - otherwise we may + * simply cause a skew in allocations. + * + * Clamp this loop to a single pass (nnodes) to keep the walk + * bounded by node count. + */ + while (target && nnodes-- && nid < MAX_NUMNODES) { /* detect system default usage */ weight = table ? table[nid] : 1; if (target < weight) break; target -= weight; - nid = next_node_in(nid, nodemask); + nid = next_node_in(nid, pol->nodes); } rcu_read_unlock(); + + /* the mask emptied under the walk */ + if (nid >= MAX_NUMNODES) + return numa_node_id(); return nid; } @@ -2241,18 +2261,23 @@ static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx) */ static unsigned int interleave_nid(struct mempolicy *pol, pgoff_t ilx) { - nodemask_t nodemask; unsigned int target, nnodes; int i; int nid; - nnodes = read_once_policy_nodemask(pol, &nodemask); + nnodes = nodes_weight(pol->nodes); if (!nnodes) return numa_node_id(); target = ilx % nnodes; - nid = first_node(nodemask); - for (i = 0; i < target; i++) - nid = next_node(nid, nodemask); + nid = first_node(pol->nodes); + + /* A concurrent cpuset rebind may cause us to see an empty nodemask */ + for (i = 0; i < target && nid < MAX_NUMNODES; i++) + nid = next_node_in(nid, pol->nodes); + + /* the mask emptied under the walk */ + if (nid >= MAX_NUMNODES) + return numa_node_id(); return nid; } @@ -2592,6 +2617,7 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp, struct mempolicy *pol, unsigned long nr_pages, struct page **page_array) { + unsigned int cpuset_mems_cookie; int nodes; unsigned long nr_pages_per_node; int delta; @@ -2599,7 +2625,16 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp, unsigned long nr_allocated; unsigned long total_allocated = 0; - nodes = nodes_weight(pol->nodes); + /* count the nodes, retry if a rebind happened during the read */ + do { + cpuset_mems_cookie = read_mems_allowed_begin(); + nodes = nodes_weight(pol->nodes); + } while (read_mems_allowed_retry(cpuset_mems_cookie)); + + /* if the nodemask has become invalid, we cannot do anything */ + if (!nodes) + return 0; + nr_pages_per_node = nr_pages / nodes; delta = nr_pages - nodes * nr_pages_per_node; @@ -2634,10 +2669,10 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, unsigned long nr_allocated = 0; unsigned long rounds; unsigned long node_pages, delta; - u8 *weights, weight; + struct srcu_ctr __percpu *scp; + u8 *table, weight; unsigned int weight_total = 0; unsigned long rem_pages = nr_pages; - nodemask_t nodes; int nnodes, node; int resume_node = MAX_NUMNODES - 1; u8 resume_weight = 0; @@ -2647,10 +2682,10 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, if (!nr_pages) return 0; - /* read the nodes onto the stack, retry if done during rebind */ + /* count the nodes, retry if a rebind happened during the read */ do { cpuset_mems_cookie = read_mems_allowed_begin(); - nnodes = read_once_policy_nodemask(pol, &nodes); + nnodes = nodes_weight(pol->nodes); } while (read_mems_allowed_retry(cpuset_mems_cookie)); /* if the nodemask has become invalid, we cannot do anything */ @@ -2660,7 +2695,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, /* Continue allocating from most recent node and adjust the nr_pages */ node = me->il_prev; weight = me->il_weight; - if (weight && node_isset(node, nodes)) { + if (weight && node_isset(node, pol->nodes)) { node_pages = min(rem_pages, weight); nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, page_array); @@ -2678,25 +2713,18 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, me->il_weight = 0; prev_node = node; - /* create a local copy of node weights to operate on outside rcu */ - weights = kmalloc(nr_node_ids, gfp & GFP_RECLAIM_MASK); - if (!weights) - return total_allocated; - - rcu_read_lock(); - state = rcu_dereference(wi_state); - if (state) { - memcpy(weights, state->iw_table, nr_node_ids * sizeof(u8)); - rcu_read_unlock(); - } else { - rcu_read_unlock(); - for (i = 0; i < nr_node_ids; i++) - weights[i] = 1; - } + /* The page allocator may sleep, pin the weight table with SRCU */ + scp = srcu_read_lock_fast(&wi_srcu); + state = srcu_dereference(wi_state, &wi_srcu); + table = state ? state->iw_table : NULL; /* calculate total, detect system default usage */ - for_each_node_mask(node, nodes) - weight_total += weights[node]; + for_each_node_mask(node, pol->nodes) + weight_total += table ? table[node] : 1; + + /* the mask emptied since it was counted */ + if (!weight_total) + goto out; /* * Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls. @@ -2707,11 +2735,15 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, */ rounds = rem_pages / weight_total; delta = rem_pages % weight_total; - resume_node = next_node_in(prev_node, nodes); - resume_weight = weights[resume_node]; + resume_node = next_node_in(prev_node, pol->nodes); + if (resume_node >= MAX_NUMNODES) + goto out; + resume_weight = table ? table[resume_node] : 1; for (i = 0; i < nnodes; i++) { - node = next_node_in(prev_node, nodes); - weight = weights[node]; + node = next_node_in(prev_node, pol->nodes); + if (node >= MAX_NUMNODES) + break; + weight = table ? table[node] : 1; node_pages = weight * rounds; /* If a delta exists, add this node's portion of the delta */ if (delta > weight) { @@ -2727,6 +2759,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, /* node_pages can be 0 if an allocation fails and rounds == 0 */ if (!node_pages) break; + /* a rebind can invalidate the counts: never overrun page_array */ + node_pages = min(node_pages, nr_pages - total_allocated); nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, page_array); page_array += nr_allocated; @@ -2737,7 +2771,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, } me->il_prev = resume_node; me->il_weight = resume_weight; - kfree(weights); +out: + srcu_read_unlock_fast(&wi_srcu, scp); return total_allocated; } @@ -3663,10 +3698,7 @@ static ssize_t node_store(struct kobject *kobj, struct kobj_attribute *attr, rcu_assign_pointer(wi_state, new_wi_state); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); return count; } @@ -3732,10 +3764,7 @@ static ssize_t weighted_interleave_auto_store(struct kobject *kobj, update_wi_state: rcu_assign_pointer(wi_state, new_wi_state); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); return count; } @@ -3779,10 +3808,7 @@ static void wi_state_free(void) rcu_assign_pointer(wi_state, NULL); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); } static struct kobj_attribute wi_auto_attr = { diff --git a/mm/migrate.c b/mm/migrate.c index 15b45832bcfa..7e3a81f06974 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -776,8 +776,9 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) { int cpupid; - if (folio_test_referenced(folio)) - folio_set_referenced(newfolio); + /* Copy the reference state, including PG_referenced */ + folio_migrate_lru_refs(newfolio, folio); + if (folio_test_uptodate(folio)) folio_mark_uptodate(newfolio); if (folio_test_clear_active(folio)) { @@ -807,7 +808,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) if (folio_test_idle(folio)) folio_set_idle(newfolio); - folio_migrate_refs(newfolio, folio); /* * Copy NUMA information to the new page, to prevent over-eager * future migrations of this same page. @@ -835,7 +835,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) */ if (folio_test_swapcache(folio)) folio_clear_swapcache(folio); - folio_clear_private(folio); /* page->private contains hugetlb specific flags */ if (!folio_test_hugetlb(folio)) @@ -1327,7 +1326,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, * free the metadata, so the page can be freed. */ if (!src->mapping) { - if (folio_test_private(src)) { + if (folio_has_attached_private(src)) { try_to_free_buffers(src); goto out; } diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 009bfa8b212d..0c437004329d 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -1398,9 +1398,9 @@ static unsigned long migrate_device_pfn_lock(unsigned long pfn) * @start: starting pfn in the range to migrate. * @npages: number of pages to migrate. * - * migrate_vma_setup() is similar in concept to migrate_vma_setup() except that - * instead of looking up pages based on virtual address mappings a range of - * device pfns that should be migrated to system memory is used instead. + * migrate_device_range() is similar in concept to migrate_vma_setup(), except + * that instead of looking up pages based on virtual address mappings, a range + * of device pfns that should be migrated to system memory is used. * * This is useful when a driver needs to free device memory but doesn't know the * virtual mappings of every page that may be in device memory. For example this diff --git a/mm/mincore.c b/mm/mincore.c index ff4ac8281768..0fe50f8a7e62 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -71,7 +71,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) */ if (shmem) { si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return 0; } folio = swap_cache_get_folio(entry); @@ -235,24 +235,22 @@ static const struct mm_walk_ops mincore_walk_ops = { .pmd_entry = mincore_pte_range, .pte_hole = mincore_unmapped_range, .hugetlb_entry = mincore_hugetlb, - .walk_lock = PGWALK_RDLOCK, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, }; /* * Do a chunk of "sys_mincore()". We've already checked - * all the arguments, we hold the mmap semaphore: we should + * all the arguments, we hold the VMA read lock: we should * just return the amount of info we're asked for. */ -static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec) +static long do_mincore(struct vm_area_struct *vma, unsigned long addr, + unsigned long pages, unsigned char *vec) { - struct vm_area_struct *vma; - unsigned long end; + unsigned long end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); int err; - vma = vma_lookup(current->mm, addr); - if (!vma) - return -ENOMEM; - end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); + vma_assert_locked(vma); + if (!can_do_mincore(vma)) { unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE); memset(vec, 1, pages); @@ -319,13 +317,20 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, retval = 0; while (pages) { + struct vm_area_struct *vma; + + vma = vma_start_read_unlocked(current->mm, start); + if (!vma) { + retval = -ENOMEM; + break; + } + /* * Do at most PAGE_SIZE entries per iteration, due to * the temporary buffer size. */ - mmap_read_lock(current->mm); - retval = do_mincore(start, min(pages, PAGE_SIZE), tmp); - mmap_read_unlock(current->mm); + retval = do_mincore(vma, start, min(pages, PAGE_SIZE), tmp); + vma_end_read(vma); if (retval <= 0) break; diff --git a/mm/mm_init.c b/mm/mm_init.c index 1533aebafb68..56bb4567a494 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -677,21 +678,33 @@ static inline void fixup_hashdist(void) static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ -#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) static __meminit void pageblock_migratetype_init_range(unsigned long pfn, - unsigned long nr_pages, int migratetype, bool atomic) + unsigned long nr_pages, int migratetype, bool isolate, bool atomic) { const unsigned long end = pfn + nr_pages; for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) { enum migratetype mt = kho_scratch_migratetype(pfn, migratetype); - init_pageblock_migratetype(pfn_to_page(pfn), mt, false); - if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION)) + init_pageblock_migratetype(pfn_to_page(pfn), mt, isolate); + if (!atomic && IS_ALIGNED(pfn, PFN_DOWN(SZ_1G))) cond_resched(); } } -#endif + +struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid) +{ + pg_data_t *pgdat = NODE_DATA(nid); + + for (enum zone_type zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + struct zone *zone = &pgdat->node_zones[zone_type]; + + if (zone_spans_pfn(zone, pfn)) + return zone; + } + + return NULL; +} #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) @@ -751,20 +764,14 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) static void __meminit __init_deferred_page(unsigned long pfn, int nid) { - pg_data_t *pgdat = NODE_DATA(nid); - int zid; + struct zone *zone; if (early_page_initialised(pfn, nid)) return; - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - struct zone *zone = &pgdat->node_zones[zid]; - - if (zone_spans_pfn(zone, pfn)) - break; - } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); - + zone = pfn_to_zone(pfn, nid); + __init_single_page(pfn_to_page(pfn), pfn, + zone ? zone_idx(zone) : MAX_NR_ZONES, nid); if (pageblock_aligned(pfn)) { enum migratetype mt = kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); @@ -886,6 +893,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone } } + /* + * Vmemmap-optimizable PFNs are backed by shared tail struct pages, + * which have already been initialized during vmemmap population. + */ + if (vmemmap_optimizable_pfn(pfn)) { + const unsigned int order = pfn_to_section_compound_order(pfn); + + pfn = min(ALIGN(pfn, 1UL << order), end_pfn); + continue; + } + page = pfn_to_page(pfn); __init_single_page(page, pfn, zone, nid); if (context == MEMINIT_HOTPLUG) { @@ -897,19 +915,13 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone __SetPageOffline(page); } - /* - * Usually, we want to mark the pageblock MIGRATE_MOVABLE, - * such that unmovable allocations won't be scattered all - * over the place during system boot. - */ - if (pageblock_aligned(pfn)) { - enum migratetype mt = kho_scratch_migratetype(pfn, migratetype); - - init_pageblock_migratetype(page, mt, isolate_pageblock); + if (pageblock_aligned(pfn)) cond_resched(); - } pfn++; } + + pageblock_migratetype_init_range(start_pfn, pfn - start_pfn, migratetype, + isolate_pageblock, /* atomic */ false); } static void __init memmap_init_zone_range(struct zone *zone, @@ -1000,13 +1012,9 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, page->zone_device_data = NULL; /* - * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released - * directly to the driver page allocator which will set the page count - * to 1 when allocating the page. - * - * MEMORY_TYPE_GENERIC and MEMORY_TYPE_FS_DAX pages automatically have - * their refcount reset to one whenever they are freed (ie. after - * their refcount drops to 0). + * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free + * path. The remaining ZONE_DEVICE types start from 0 here and raise + * the count again when the allocator or driver hands the page out. */ switch (pgmap->type) { case MEMORY_DEVICE_FS_DAX: @@ -1021,6 +1029,17 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, } } +static void zone_device_page_init_from_template(struct page *page, + unsigned long pfn, struct page *template) +{ + set_page_section_from_pfn(template, pfn); +#ifdef WANT_PAGE_VIRTUAL + if (!is_highmem_idx(ZONE_DEVICE)) + set_page_address(template, __va(pfn << PAGE_SHIFT)); +#endif + memcpy_nontemporal(page, template, sizeof(*page)); +} + /* * With compound page geometry and when struct pages are stored in ram most * tail pages are reused. Consequently, the amount of unique struct pages to @@ -1030,19 +1049,14 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, * of an altmap. See vmemmap_populate_compound_pages(). */ static inline unsigned long compound_nr_pages(unsigned long pfn, - struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - /* - * If DAX memory is hot-plugged into an unoccupied subsection - * of an early section, the unoptimized boot memmap is reused. - * See section_activate(). - */ - if (early_section(__pfn_to_section(pfn)) || - !vmemmap_can_optimize(altmap, pgmap)) + const struct mem_section *ms = __pfn_to_section(pfn); + + if (!section_vmemmap_optimizable(ms)) return pgmap_vmemmap_nr(pgmap); - return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); + return VMEMMAP_OPTIMIZATION_PAGES * (PAGE_SIZE / sizeof(struct page)); } static void __ref memmap_init_compound(struct page *head, @@ -1053,6 +1067,8 @@ static void __ref memmap_init_compound(struct page *head, { unsigned long pfn, end_pfn = head_pfn + nr_pages; unsigned int order = pgmap->vmemmap_shift; + struct page template; + struct page *page; /* * We have to initialize the pages, including setting up page links. @@ -1061,13 +1077,23 @@ static void __ref memmap_init_compound(struct page *head, * the pages in the same go. */ __SetPageHead(head); - for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { - struct page *page = pfn_to_page(pfn); - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); - prep_compound_tail(page, head, order); - set_page_count(page, 0); - } + /* + * All tails of the same compound page share the state established by + * prep_compound_tail(). Reuse one tail template for the whole range and + * refresh only the PFN-dependent fields in that template before each copy. + */ + pfn = head_pfn + 1; + page = pfn_to_page(pfn); + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + prep_compound_tail(page, head, order); + set_page_count(page, 0); + memcpy(&template, page, sizeof(*page)); + + /* Initialize the remaining tail pages from template. */ + for (pfn = head_pfn + 2; pfn < end_pfn; pfn++) + zone_device_page_init_from_template(pfn_to_page(pfn), pfn, + &template); prep_compound_head(head, order); } @@ -1083,6 +1109,8 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long zone_idx = zone_idx(zone); unsigned long start = jiffies; int nid = pgdat->node_id; + struct page template; + struct page *page; if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) return; @@ -1097,10 +1125,28 @@ void __ref memmap_init_zone_device(struct zone *zone, nr_pages = end_pfn - start_pfn; } - for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { - struct page *page = pfn_to_page(pfn); + if (!nr_pages) + return; - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + /* + * Seed the reusable head-page template from the first real struct + * page. The normal page-init and refcount helpers must operate on + * a real memmap entry rather than a stack object. + */ + pfn = start_pfn; + page = pfn_to_page(pfn); + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + memcpy(&template, page, sizeof(*page)); + if (pfns_per_compound != 1) + memmap_init_compound(page, pfn, zone_idx, nid, pgmap, + compound_nr_pages(pfn, pgmap)); + pfn += pfns_per_compound; + + /* Initialize the remaining head pages from template. */ + for (; pfn < end_pfn; pfn += pfns_per_compound) { + page = pfn_to_page(pfn); + + zone_device_page_init_from_template(page, pfn, &template); if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); @@ -1109,10 +1155,11 @@ void __ref memmap_init_zone_device(struct zone *zone, continue; memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, altmap, pgmap)); + compound_nr_pages(pfn, pgmap)); } - pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false); + pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, + /* isolate */ false, /* atomic */ false); pr_debug("%s initialised %lu pages in %ums\n", __func__, nr_pages, jiffies_to_msecs(jiffies - start)); @@ -1921,7 +1968,8 @@ static void __init deferred_free_pages(unsigned long pfn, if (!nr_pages) return; - pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE, true); + pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE, + /* isolate */ false, /* atomic */ true); page = pfn_to_page(pfn); @@ -2628,6 +2676,7 @@ void __init mm_core_init_early(void) { kho_memory_init_early(); + sparse_sections_init(); free_area_init(); hugetlb_cma_reserve(); diff --git a/mm/mm_init.h b/mm/mm_init.h index 39f75df9be1c..c9fc35e7e9f1 100644 --- a/mm/mm_init.h +++ b/mm/mm_init.h @@ -39,6 +39,7 @@ void memmap_init_range(unsigned long size, int nid, unsigned long zone, enum meminit_context context, struct vmem_altmap *altmap, int migratetype, bool isolate_pageblock); +struct zone *pfn_to_zone(unsigned long pfn, int nid); #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c index 898c2ef1e958..2f94ee0fdee2 100644 --- a/mm/mmap_lock.c +++ b/mm/mmap_lock.c @@ -43,9 +43,6 @@ void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write) EXPORT_SYMBOL(__mmap_lock_do_trace_released); #endif /* CONFIG_TRACING */ -#ifdef CONFIG_MMU -#ifdef CONFIG_PER_VMA_LOCK - /* State shared across __vma_[start, end]_exclude_readers. */ struct vma_exclude_readers_state { /* Input parameters. */ @@ -299,6 +296,8 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, MA_STATE(mas, &mm->mm_mt, address, address); struct vm_area_struct *vma; + if (!IS_ENABLED(CONFIG_MMU)) + return NULL; retry: rcu_read_lock(); vma = mas_walk(&mas); @@ -341,6 +340,41 @@ inval: return NULL; } +/** + * vma_start_read_unlocked() - Find the VMA covering 'address' and read-lock it. + * @mm: the mm_struct of the address space to search + * @address: address that the vma should contain + * + * The fast path does not take mmap_lock. Waits for writers to finish if the + * VMA is being modified by taking mmap_lock. + * Use when mmap_lock is not held, otherwise use vma_start_read_locked(). + * Nothing prevents VMAs being unmapped/mapped before or after the VMA is + * looked up, if a stronger guarantee is required, take an mmap_lock. + * + * Return: If a VMA exists which spans @address, return that VMA, read-locked. + * If no VMA is mapped there or, very unlikely, a reference count overflow + * occurred, return NULL. + */ +struct vm_area_struct *vma_start_read_unlocked(struct mm_struct *mm, + unsigned long address) +{ + struct vm_area_struct *vma; + + /* Fast path: return stable VMA covering 'address': */ + vma = lock_vma_under_rcu(mm, address); + if (vma) + return vma; + + /* Slow path: preclude VMA writers by temporarily getting mmap read lock. */ + mmap_read_lock(mm); + vma = vma_lookup(mm, address); + if (vma && !vma_start_read_locked(vma)) + vma = NULL; + mmap_read_unlock(mm); + + return vma; +} + static struct vm_area_struct *lock_next_vma_under_mmap_lock(struct mm_struct *mm, struct vma_iterator *vmi, unsigned long from_addr) @@ -431,7 +465,6 @@ fallback: return vma; } -#endif /* CONFIG_PER_VMA_LOCK */ #ifdef CONFIG_LOCK_MM_AND_FIND_VMA #include @@ -548,23 +581,3 @@ fail: return NULL; } #endif /* CONFIG_LOCK_MM_AND_FIND_VMA */ - -#else /* CONFIG_MMU */ - -/* - * At least xtensa ends up having protection faults even with no - * MMU.. No stack expansion, at least. - */ -struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, - unsigned long addr, struct pt_regs *regs) -{ - struct vm_area_struct *vma; - - mmap_read_lock(mm); - vma = vma_lookup(mm, addr); - if (!vma) - mmap_read_unlock(mm); - return vma; -} - -#endif /* CONFIG_MMU */ diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 3985d856de7f..2a72a9686773 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -218,8 +218,6 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_ #endif /* MMU_GATHER_NO_GATHER */ -#ifdef CONFIG_MMU_GATHER_TABLE_FREE - static void __tlb_remove_table_free(struct mmu_table_batch *batch) { int i; @@ -230,10 +228,8 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch) free_page((unsigned long)batch); } -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE - /* - * Semi RCU freeing of the page directories. + * RCU freeing of the page directories. * * This is needed by some architectures to implement software pagetable walkers. * @@ -259,13 +255,13 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch) * means. * * What we do is batch the freed directory pages (tables) and RCU free them. - * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling - * holds off grace periods. + * Disabling IRQs or preemption holds off RCU grace periods, so this protects + * both rcu_read_lock() and IRQ-disabling walkers. * * However, in order to batch these pages we need to allocate storage, this * allocation is deep inside the MM code and can thus easily fail on memory - * pressure. To guarantee progress we fall back to single table freeing, see - * the implementation of tlb_remove_table_one(). + * pressure. To guarantee progress we fall back to single table freeing, which + * is also RCU-deferred - see the implementation of tlb_remove_table_one(). * */ @@ -315,15 +311,6 @@ void tlb_remove_table_sync_rcu(void) synchronize_rcu(); } -#else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */ - -static void tlb_remove_table_free(struct mmu_table_batch *batch) -{ - __tlb_remove_table_free(batch); -} - -#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ - /* * If we want tlb_remove_table() to imply TLB invalidates. */ @@ -403,13 +390,6 @@ static inline void tlb_table_init(struct mmu_gather *tlb) tlb->batch = NULL; } -#else /* !CONFIG_MMU_GATHER_TABLE_FREE */ - -static inline void tlb_table_flush(struct mmu_gather *tlb) { } -static inline void tlb_table_init(struct mmu_gather *tlb) { } - -#endif /* CONFIG_MMU_GATHER_TABLE_FREE */ - static void tlb_flush_mmu_free(struct mmu_gather *tlb) { tlb_table_flush(tlb); diff --git a/mm/nommu.c b/mm/nommu.c index 498e01ee40b0..ed44510e3770 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1674,6 +1674,9 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, struct vm_area_struct *vma; int write = gup_flags & FOLL_WRITE; + if (addr + len < addr) + return 0; + if (mmap_read_lock_killable(mm)) return 0; @@ -1727,9 +1730,6 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in { struct mm_struct *mm; - if (addr + len < addr) - return 0; - mm = get_task_mm(tsk); if (!mm) return 0; diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 5f372f6e26fa..5d48bd862c27 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -230,7 +230,7 @@ long oom_badness(struct task_struct *p, unsigned long totalpages) task_unlock(p); /* Normalize to oom_score_adj units */ - adj *= totalpages / 1000; + adj *= totalpages / OOM_SCORE_ADJ_MAX; points += adj; return points; @@ -267,18 +267,11 @@ static enum oom_constraint constrained_alloc(struct oom_control *oc) if (!oc->zonelist) return CONSTRAINT_NONE; - /* - * Reach here only when __GFP_NOFAIL is used. So, we should avoid - * to kill current.We have to random task kill in this case. - * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now. - */ - if (oc->gfp_mask & __GFP_THISNODE) - return CONSTRAINT_NONE; /* - * This is not a __GFP_THISNODE allocation, so a truncated nodemask in - * the page allocator means a mempolicy is in effect. Cpuset policy - * is enforced in get_page_from_freelist(). + * A truncated nodemask in the page allocator means a mempolicy + * is in effect. Cpuset policy is enforced in + * get_page_from_freelist(). */ if (oc->nodemask && !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) { diff --git a/mm/page-writeback.c b/mm/page-writeback.c index eeab25d6ce36..499a35473e4f 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2705,7 +2705,7 @@ bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio) if (folio_test_set_dirty(folio)) return false; - __folio_mark_dirty(folio, mapping, !folio_test_private(folio)); + __folio_mark_dirty(folio, mapping, !folio_has_attached_private(folio)); if (mapping->host) { /* !PageAnon && !swapper_space */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 12fac9084c48..ff7b9acf0144 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -613,31 +614,13 @@ static inline bool __maybe_unused bad_range(struct zone *zone, struct page *page } #endif +/* Allow a burst of 60 reports per minute */ +static DEFINE_RATELIMIT_STATE(bad_page_ratelimit, 60 * HZ, 60); + static void bad_page(struct page *page, const char *reason) { - static unsigned long resume; - static unsigned long nr_shown; - static unsigned long nr_unshown; - - /* - * Allow a burst of 60 reports, then keep quiet for that minute; - * or allow a steady drip of one report per second. - */ - if (nr_shown == 60) { - if (time_before(jiffies, resume)) { - nr_unshown++; - goto out; - } - if (nr_unshown) { - pr_alert( - "BUG: Bad page state: %lu messages suppressed\n", - nr_unshown); - nr_unshown = 0; - } - nr_shown = 0; - } - if (nr_shown++ == 0) - resume = jiffies + 60 * HZ; + if (!__ratelimit(&bad_page_ratelimit)) + goto out; pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", current->comm, page_to_pfn(page)); @@ -2178,10 +2161,17 @@ static inline bool boost_watermark(struct zone *zone) if (!watermark_boost_factor) return false; + + /* + * A kdump capture kernel exits before a boost can pay off, while + * the raised watermark can exceed the memory left for the dump. + */ + if (is_kdump_kernel()) + return false; + /* * Don't bother in zones that are unlikely to produce results. - * On small machines, including kdump capture kernels running - * in a small area, boosting the watermark can cause an out of + * On small machines, boosting the watermark can cause an out of * memory situation immediately. */ if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) @@ -4804,7 +4794,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, if (unlikely(nofail)) { /* - * Also we don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM, + * We don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM, * otherwise, we may result in lockup. */ WARN_ON_ONCE(!can_direct_reclaim); @@ -5207,6 +5197,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ gfp &= gfp_allowed_mask; + gfp = current_gfp_context(gfp); if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &gfp, &alloc_flags)) goto out; @@ -5279,6 +5270,8 @@ retry_this_zone: nr_account++; prep_new_page(page, 0, gfp, ALLOC_DEFAULT); + trace_mm_page_alloc(page, 0, gfp, ac.migratetype); + kmsan_alloc_page(page, 0, gfp & ~__GFP_RECLAIM); set_page_refcounted(page); page_array[nr_populated++] = page; } @@ -5797,9 +5790,10 @@ static int numa_zonelist_order_handler(const struct ctl_table *table, int write, static int node_load[MAX_NUMNODES]; /** - * find_next_best_node - find the next node that should appear in a given node's fallback list + * find_next_best_node_in - find the next node that should appear in a given node's fallback list * @node: node whose fallback list we're appending * @used_node_mask: nodemask_t of already used nodes + * @candidates: nodemask_t of nodes eligible for selection * * We use a number of factors to determine which is the next node that should * appear on a given node's fallback list. The node should not have appeared @@ -5811,7 +5805,8 @@ static int node_load[MAX_NUMNODES]; * * Return: node id of the found node or %NUMA_NO_NODE if no node is found. */ -int find_next_best_node(int node, nodemask_t *used_node_mask) +int find_next_best_node_in(int node, nodemask_t *used_node_mask, + const nodemask_t *candidates) { int n, val; int min_val = INT_MAX; @@ -5821,12 +5816,12 @@ int find_next_best_node(int node, nodemask_t *used_node_mask) * Use the local node if we haven't already, but for memoryless local * node, we should skip it and fall back to other nodes. */ - if (!node_isset(node, *used_node_mask) && node_state(node, N_MEMORY)) { + if (!node_isset(node, *used_node_mask) && node_isset(node, *candidates)) { node_set(node, *used_node_mask); return node; } - for_each_node_state(n, N_MEMORY) { + for_each_node_mask(n, *candidates) { /* Don't want a node to appear more than once */ if (node_isset(n, *used_node_mask)) @@ -5859,31 +5854,6 @@ int find_next_best_node(int node, nodemask_t *used_node_mask) } -/* - * Build zonelists ordered by node and zones within node. - * This results in maximum locality--normal zone overflows into local - * DMA zone, if any--but risks exhausting DMA zone. - */ -static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, - unsigned nr_nodes) -{ - struct zoneref *zonerefs; - int i; - - zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; - - for (i = 0; i < nr_nodes; i++) { - int nr_zones; - - pg_data_t *node = NODE_DATA(node_order[i]); - - nr_zones = build_zonerefs_node(node, zonerefs); - zonerefs += nr_zones; - } - zonerefs->zone = NULL; - zonerefs->zone_idx = 0; -} - /* * Build __GFP_THISNODE zonelists */ @@ -5899,19 +5869,24 @@ static void build_thisnode_zonelists(pg_data_t *pgdat) zonerefs->zone_idx = 0; } -static void build_zonelists(pg_data_t *pgdat) +/* + * Build one zonelist ordered by node and zones within node. This results in + * maximum locality--normal zone overflows into local DMA zone, if any--but + * risks exhausting DMA zone. + */ +static void build_node_zonelist(pg_data_t *pgdat, const nodemask_t *candidates, + int zlidx) { - static int node_order[MAX_NUMNODES]; - int node, nr_nodes = 0; + struct zoneref *zonerefs = pgdat->node_zonelists[zlidx]._zonerefs; nodemask_t used_mask = NODE_MASK_NONE; - int local_node, prev_node; + int local_node = pgdat->node_id; + int prev_node = local_node; + int node; - /* NUMA-aware ordering of nodes */ - local_node = pgdat->node_id; - prev_node = local_node; + pr_info("Fallback order for Node %d: ", local_node); - memset(node_order, 0, sizeof(node_order)); - while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { + while ((node = find_next_best_node_in(local_node, &used_mask, + candidates)) >= 0) { /* * We don't want to pressure a particular node. * So adding penalty to the first node in same @@ -5921,18 +5896,22 @@ static void build_zonelists(pg_data_t *pgdat) node_distance(local_node, prev_node)) node_load[node] += 1; - node_order[nr_nodes++] = node; + zonerefs += build_zonerefs_node(NODE_DATA(node), zonerefs); + pr_cont("%d ", node); prev_node = node; } - build_zonelists_in_node_order(pgdat, node_order, nr_nodes); - build_thisnode_zonelists(pgdat); - pr_info("Fallback order for Node %d: ", local_node); - for (node = 0; node < nr_nodes; node++) - pr_cont("%d ", node_order[node]); + zonerefs->zone = NULL; + zonerefs->zone_idx = 0; pr_cont("\n"); } +static void build_zonelists(pg_data_t *pgdat) +{ + build_node_zonelist(pgdat, &node_states[N_MEMORY], ZONELIST_FALLBACK); + build_thisnode_zonelists(pgdat); +} + #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* * Return node id of node used for "local" allocations. @@ -7978,7 +7957,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, /* * Watermarks have not been initialized yet. * - * Accepting one MAX_ORDER page to ensure progress. + * Accepting one MAX_PAGE_ORDER page to ensure progress. */ if (!wmark) return try_to_accept_memory_one(zone); diff --git a/mm/page_counter.c b/mm/page_counter.c index 661e0f2a5127..9167ffd1380c 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,26 @@ static void propagate_protected_usage(struct page_counter *c, } } +/** + * page_counter_margin - remaining usable space within hierarchical limits + * @counter: counter + * + * Return: The minimum value of max minus usage across @counter and all of + * its ancestors. The value may be negative during a concurrent charge. + */ +long page_counter_margin(const struct page_counter *counter) +{ + long margin = PAGE_COUNTER_MAX; + + do { + long m = READ_ONCE(counter->max) - page_counter_read(counter); + + margin = min(margin, m); + } while ((counter = counter->parent)); + + return margin; +} + /** * page_counter_cancel - take pages out of the local counter * @counter: counter @@ -356,7 +377,8 @@ static unsigned long effective_protection(unsigned long usage, * otherwise get a smaller chunk than what they claimed. */ if (siblings_protected > parent_effective) - return protected * parent_effective / siblings_protected; + return mul_u64_u64_div_u64(protected, parent_effective, + siblings_protected); /* * Ok, utilized protection of all children is within what the @@ -397,13 +419,20 @@ static unsigned long effective_protection(unsigned long usage, if (parent_effective > siblings_protected && parent_usage > siblings_protected && usage > protected) { - unsigned long unclaimed; + unsigned long parent_unclaimed, parent_unprotected, unprotected; - unclaimed = parent_effective - siblings_protected; - unclaimed *= usage - protected; - unclaimed /= parent_usage - siblings_protected; + parent_unclaimed = parent_effective - siblings_protected; + parent_unprotected = parent_usage - siblings_protected; - ep += unclaimed; + /* + * The usages aren't read atomically, so a child can transiently + * appear to use more than its parent, making the ratio exceed 1 + * and the quotient overflow 64 bits (#DE on x86). Cap it. + */ + unprotected = min(usage - protected, parent_unprotected); + + ep += mul_u64_u64_div_u64(parent_unclaimed, unprotected, + parent_unprotected); } return ep; diff --git a/mm/page_io.c b/mm/page_io.c index 88962571cb93..0808808f0309 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -160,7 +160,7 @@ static void swap_zeromap_folio_set(struct folio *folio) struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio); int nr_pages = folio_nr_pages(folio); struct swap_cluster_info *ci; - swp_entry_t entry; + swp_entry_t entry = folio->swap; unsigned int i; VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); @@ -168,8 +168,8 @@ static void swap_zeromap_folio_set(struct folio *folio) ci = swap_cluster_get_and_lock(folio); for (i = 0; i < folio_nr_pages(folio); i++) { - entry = page_swap_entry(folio_page(folio, i)); __swap_table_set_zero(ci, swp_cluster_offset(entry)); + entry.val++; } swap_cluster_unlock(ci); @@ -183,7 +183,7 @@ static void swap_zeromap_folio_set(struct folio *folio) static void swap_zeromap_folio_clear(struct folio *folio) { struct swap_cluster_info *ci; - swp_entry_t entry; + swp_entry_t entry = folio->swap; unsigned int i; VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); @@ -191,8 +191,8 @@ static void swap_zeromap_folio_clear(struct folio *folio) ci = swap_cluster_get_and_lock(folio); for (i = 0; i < folio_nr_pages(folio); i++) { - entry = page_swap_entry(folio_page(folio, i)); __swap_table_clear_zero(ci, swp_cluster_offset(entry)); + entry.val++; } swap_cluster_unlock(ci); } @@ -209,7 +209,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) goto out_unlock; /* - * Arch code may have to preserve more data than just the page + * Arch code may have to preserve more data than just the folio * contents, e.g. memory tags. */ ret = arch_prepare_to_swap(folio); @@ -220,7 +220,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) /* * Use the swap table zero mark to avoid doing IO for zero-filled - * pages. The zero mark is protected by the cluster lock, which is + * folios. The zero mark is protected by the cluster lock, which is * acquired internally by swap_zeromap_folio_set/clear. */ if (is_folio_zero_filled(folio)) { @@ -248,7 +248,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) } rcu_read_unlock(); - __swap_writepage(ctx, folio); + __swap_writeout(ctx, folio); return 0; out_unlock: folio_unlock(folio); @@ -256,12 +256,13 @@ out_unlock: } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) -static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio) +static struct cgroup_subsys_state *folio_memcg_blkg_css(const struct folio *folio) { return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys); } -static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +static bool folio_blkg_can_merge(const struct folio *folio, + const struct folio *prev_folio) { bool can_merge = true; @@ -277,7 +278,8 @@ static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) return can_merge; } -static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) +static void bio_associate_blkg_from_folio(struct bio *bio, + const struct folio *folio) { struct cgroup_subsys_state *css; @@ -294,11 +296,15 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) css_put(css); } #else -static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +static bool folio_blkg_can_merge(const struct folio *folio, + const struct folio *prev_folio) { return true; } -#define bio_associate_blkg_from_page(bio, folio) do { } while (0) +static void bio_associate_blkg_from_folio(struct bio *bio, + const struct folio *folio) +{ +} #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ static mempool_t *sio_pool; @@ -367,7 +373,7 @@ static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw) } } -void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio) +void __swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); @@ -497,13 +503,13 @@ static void swap_write_end(struct swap_iocb *sio, bool failed) int p; for (p = 0; p < sio->nr_bvecs; p++) { - struct page *page = sio->bvecs[p].bv_page; + struct folio *folio = bvec_folio(&sio->bvecs[p]); if (failed) { - set_page_dirty(page); - ClearPageReclaim(page); + folio_mark_dirty(folio); + folio_clear_reclaim(folio); } - end_page_writeback(page); + folio_end_writeback(folio); } mempool_free(sio, sio_pool); } @@ -514,16 +520,16 @@ static void swap_fs_write_complete(struct kiocb *iocb, long ret) bool failed = ret != sio->len; if (failed) { - struct page *page = sio->bvecs[0].bv_page; + struct folio *folio = bvec_folio(&sio->bvecs[0]); /* * In the case of swap-over-nfs, this can be a temporary failure * if the system has limited memory for allocating transmit - * buffers. Mark the page dirty and avoid + * buffers. Mark the folio dirty and avoid * folio_rotate_reclaimable but rate-limit the messages. */ pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", - ret, swap_dev_pos(page_swap_entry(page))); + ret, swap_dev_pos(folio->swap)); } swap_write_end(sio, failed); @@ -596,12 +602,21 @@ static void swap_bdev_submit_write(struct swap_io_ctx *ctx) REQ_OP_WRITE | REQ_SWAP); bio->bi_iter.bi_size = sio->len; bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); - bio_associate_blkg_from_page(bio, bio_first_folio_all(bio)); + bio_associate_blkg_from_folio(bio, bio_first_folio_all(bio)); if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { submit_bio_wait(bio); end_swap_bio_write(bio); } else { + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + if (folio_test_dropbehind(bvec_folio(&sio->bvecs[p]))) { + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); + break; + } + } + bio->bi_end_io = end_swap_bio_write; submit_bio(bio); } diff --git a/mm/page_isolation.c b/mm/page_isolation.c index e5dfc7bf4944..5aaad037384e 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -388,13 +388,19 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, } if (PageBuddy(page)) { - int order = buddy_order(page); + unsigned int order = buddy_order_unsafe(page); - /* pageblock_isolate_and_move_free_pages() handled this */ - VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn); - - pfn += 1UL << order; - continue; + /* buddy_order_unsafe() is racy. Validate the order before shifting. */ + if (order <= MAX_PAGE_ORDER && + /* + * pageblock_isolate_and_move_free_pages() splits + * cross-boundary PageBuddy, verify it. + */ + pfn + (1UL << order) <= boundary_pfn) { + pfn += 1UL << order; + continue; + } + goto failed; } /* @@ -413,10 +419,28 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, if (PageCompound(page)) { struct page *head = compound_head(page); unsigned long head_pfn = page_to_pfn(head); - unsigned long nr_pages = compound_nr(head); + unsigned int order = compound_order(head); + unsigned long nr_pages; + + /* compound_order() is racy. Cap it at MAX_FOLIO_ORDER. */ + if (order > MAX_FOLIO_ORDER) + goto failed; + + nr_pages = 1UL << order; + + /* + * compound_head() is also racy, so the derived head_pfn + * needs additional checks to make sure it is valid. + * Otherwise, just fail the check. pfn comes from + * __first_valid_page() as a legitimate PFN, so use it to + * check head_pfn. + */ + if (head_pfn > pfn || !IS_ALIGNED(head_pfn, nr_pages) || + pfn - head_pfn >= nr_pages) + goto failed; if (head_pfn + nr_pages <= boundary_pfn || - PageHuge(page)) { + PageHuge(head)) { pfn = head_pfn + nr_pages; continue; } diff --git a/mm/page_owner.c b/mm/page_owner.c index fbbda7ba914b..cfc31c92d765 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -307,8 +307,8 @@ static inline void __update_page_owner_free_handle(struct page *page, page_owner->free_handle = handle; } page_owner->free_ts_nsec = free_ts_nsec; - page_owner->free_pid = current->pid; - page_owner->free_tgid = current->tgid; + page_owner->free_pid = pid; + page_owner->free_tgid = tgid; } rcu_read_unlock(); } @@ -575,7 +575,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, } objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); - memcg = objcg ? obj_cgroup_memcg(objcg) : NULL; + memcg = obj_cgroup_memcg(objcg); if (!memcg) goto out_unlock; diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 6ffc536359cd..143a918c0bde 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -67,7 +67,7 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) struct page *page; bool anon; - if (!pfn_valid(pfn)) + if (!pfn_valid(pfn) || is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return; page = pfn_to_page(pfn); @@ -102,7 +102,7 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, struct page *page; bool anon; - if (!pfn_valid(pfn)) + if (!pfn_valid(pfn) || is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return; page = pfn_to_page(pfn); @@ -278,7 +278,7 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, if (&init_mm == mm) return; - if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { + if (!pmd_none(pmd) && !pmd_bad(pmd) && !pmd_leaf(pmd)) { pte_t *ptep = pte_offset_map(&pmd, addr); unsigned long i; diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 4e964545e5e8..28e306fdb3a5 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -142,6 +142,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) return true; } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* Returns true if the two ranges overlap. Careful to not overflow. */ static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw) { @@ -151,6 +152,12 @@ static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw) return false; return true; } +#else +static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw) +{ + return false; +} +#endif static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) { diff --git a/mm/pagewalk.c b/mm/pagewalk.c index cc07fcf50e87..7411702a37f5 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -444,7 +444,6 @@ static inline void process_mm_walk_lock(struct mm_struct *mm, static inline void process_vma_walk_lock(struct vm_area_struct *vma, enum page_walk_lock walk_lock) { -#ifdef CONFIG_PER_VMA_LOCK switch (walk_lock) { case PGWALK_WRLOCK: vma_start_write(vma); @@ -459,7 +458,6 @@ static inline void process_vma_walk_lock(struct vm_area_struct *vma, /* PGWALK_RDLOCK is handled by process_mm_walk_lock */ break; } -#endif } /* diff --git a/mm/percpu.c b/mm/percpu.c index a802d72c116f..3eff382e565a 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -100,7 +100,7 @@ /* * The slots are sorted by the size of the biggest continuous free area. - * 1-31 bytes share the same slot. + * [PCPU_MIN_ALLOC_SIZE..15] bytes share the same slot. */ #define PCPU_SLOT_BASE_SHIFT 5 /* chunks in slots below this are subject to being sidelined on failed alloc */ @@ -758,7 +758,6 @@ static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan) chunk_md->contig_hint = 0; } - bits = 0; pcpu_for_each_md_free_region(chunk, bit_off, bits) pcpu_block_update(chunk_md, bit_off, bit_off + bits); } @@ -1122,7 +1121,6 @@ static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits, return -1; bit_off = pcpu_next_hint(chunk_md, alloc_bits); - bits = 0; pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) { if (!pop_only || pcpu_is_populated(chunk, bit_off, bits, &next_off)) @@ -2819,7 +2817,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( static int group_cnt[NR_CPUS] __initdata; static struct cpumask mask __initdata; const size_t static_size = __per_cpu_end - __per_cpu_start; - int nr_groups = 1, nr_units = 0; + int nr_groups, nr_units = 0; size_t size_sum, min_unit_size, alloc_size; int upa, max_upa, best_upa; /* units_per_alloc */ int last_allocs, group, unit; @@ -2830,7 +2828,6 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( /* this function may be called multiple times */ memset(group_map, 0, sizeof(group_map)); memset(group_cnt, 0, sizeof(group_cnt)); - cpumask_clear(&mask); /* calculate size_sum and ensure dyn_size is enough for early alloc */ size_sum = PFN_ALIGN(static_size + reserved_size + @@ -3181,8 +3178,6 @@ void __init __weak pcpu_populate_pte(unsigned long addr) new = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE); pmd_populate_kernel(&init_mm, pmd, new); } - - return; } /** diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index cd227fc05d2d..26643d76bfb0 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -26,14 +26,20 @@ void pgd_clear_bad(pgd_t *pgd) { - pgd_ERROR(*pgd); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pgd_val(*pgd)); + pr_err("bad pgd %s.\n", str); pgd_clear(pgd); } #ifndef __PAGETABLE_P4D_FOLDED void p4d_clear_bad(p4d_t *p4d) { - p4d_ERROR(*p4d); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, p4d_val(*p4d)); + pr_err("bad p4d %s.\n", str); p4d_clear(p4d); } #endif @@ -41,7 +47,10 @@ void p4d_clear_bad(p4d_t *p4d) #ifndef __PAGETABLE_PUD_FOLDED void pud_clear_bad(pud_t *pud) { - pud_ERROR(*pud); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pud_val(*pud)); + pr_err("bad pud %s.\n", str); pud_clear(pud); } #endif @@ -53,7 +62,10 @@ void pud_clear_bad(pud_t *pud) */ void pmd_clear_bad(pmd_t *pmd) { - pmd_ERROR(*pmd); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pmd_val(*pmd)); + pr_err("bad pmd %s.\n", str); pmd_clear(pmd); } @@ -385,10 +397,17 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, * Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will * be read-only/read-write protected. * - * Note that free_pgtables(), used after unmapping detached vmas, or when - * exiting the whole mm, does not take page table lock before freeing a page - * table, and may not use RCU at all: "outsiders" like khugepaged should avoid - * pte_offset_map() and co once the vma is detached from mm or mm_users is zero. + * Note that free_pgtables(), used after unmapping detached vmas or when exiting + * the whole mm, does not take a page table lock before freeing a page table. + * + * As page table freeing itself is RCU-safe, page table readers can safely run + * concurrently with page table teardown. + * + * However, writers CANNOT as, without a lock being held, nothing prevents + * concurrent teardown. + * + * Also note that the PGD itself is freed at mmdrop() time, not under RCU - so + * the walker must keep the mm alive either by pinning the mm or the VMA. */ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp) diff --git a/mm/rmap.c b/mm/rmap.c index f3b21aaa34ee..5332c52909be 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -264,11 +264,9 @@ static void check_anon_vma_clone(struct vm_area_struct *dst, /* For the anon_vma to be compatible, it can only be singular. */ VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED && !list_is_singular(&src->anon_vma_chain)); -#ifdef CONFIG_PER_VMA_LOCK /* Only merging an unfaulted VMA leaves the destination attached. */ VM_WARN_ON_ONCE(operation != VMA_OP_MERGE_UNFAULTED && vma_is_attached(dst)); -#endif } static void maybe_reuse_anon_vma(struct vm_area_struct *dst, @@ -2149,7 +2147,7 @@ static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, { const bool anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); - swp_entry_t entry = page_swap_entry(page); + swp_entry_t entry = folio_page_swap_entry(folio, page); struct mm_struct *mm = vma->vm_mm; if (folio_dup_swap(folio, page) < 0) @@ -2303,11 +2301,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, VM_BUG_ON_FOLIO(!pvmw.pte, folio); address = pvmw.address; - if (folio_test_hugetlb(folio)) { - pteval = huge_ptep_get(mm, address, pvmw.pte); - } else { - pteval = ptep_get(pvmw.pte); - } + pteval = ptep_get(pvmw.pte); + if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { diff --git a/mm/secretmem.c b/mm/secretmem.c index af95b4dc943a..b3f79ad56790 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -39,7 +39,7 @@ static bool secretmem_enable __ro_after_init = 1; module_param_named(enable, secretmem_enable, bool, 0400); -MODULE_PARM_DESC(secretmem_enable, +MODULE_PARM_DESC(enable, "Enable secretmem and memfd_secret(2) system call"); static atomic_t secretmem_users; @@ -139,7 +139,7 @@ retry: goto out; } - err = set_direct_map_invalid_noflush(folio_page(folio, 0)); + err = set_direct_map_invalid_noflush(folio_page(folio, 0), 1); if (err) { secretmem_unaccount_folio(state, folio); folio_put(folio); @@ -156,7 +156,7 @@ retry: * already happened when we marked the page invalid * which guarantees that this call won't fail */ - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_put(folio); if (err == -EEXIST) goto retry; @@ -228,7 +228,7 @@ static int secretmem_migrate_folio(struct address_space *mapping, static void secretmem_free_folio(struct folio *folio) { - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_zero_segment(folio, 0, folio_size(folio)); } diff --git a/mm/shmem.c b/mm/shmem.c index 0d891ade18b5..d97fc32e7c94 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -725,51 +725,258 @@ static const char *shmem_format_huge(int huge) } #endif +static bool is_shmem_unused_huge_isolated(struct shmem_inode_info *info) +{ + + return info->shrinklist_nid == -1; +} + +static void set_shmem_unused_huge_isolated(struct shmem_inode_info *info) +{ + info->shrinklist_nid = -1; +} + +static struct obj_cgroup *shmem_get_and_clear_objcg(struct shmem_inode_info *info) +{ + struct obj_cgroup *objcg = info->shrinklist_objcg; + + info->shrinklist_objcg = NULL; + + return objcg; +} + +#ifdef CONFIG_MEMCG +static struct obj_cgroup * +shmem_unused_huge_alloc_lru(struct shmem_sb_info *sbinfo, struct folio *folio, + gfp_t gfp) +{ + int ret; + + ret = folio_memcg_list_lru_alloc(folio, &sbinfo->shrinklist, gfp); + if (ret) + return ERR_PTR(ret); + + return get_obj_cgroup_from_folio(folio); +} +#else +static struct obj_cgroup * +shmem_unused_huge_alloc_lru(struct shmem_sb_info *sbinfo, struct folio *folio, + gfp_t gfp) +{ + return NULL; +} +#endif + +static void shmem_unused_huge_lru_add(struct shmem_sb_info *sbinfo, + struct list_head *item, int nid, + struct obj_cgroup *objcg) +{ + struct mem_cgroup *memcg; + + rcu_read_lock(); + memcg = obj_cgroup_memcg(objcg); + list_lru_add(&sbinfo->shrinklist, item, nid, memcg); + rcu_read_unlock(); +} + +static void shmem_unused_huge_lru_del(struct shmem_sb_info *sbinfo, + struct list_head *item, int nid, + struct obj_cgroup *objcg) +{ + struct mem_cgroup *memcg; + + rcu_read_lock(); + memcg = obj_cgroup_memcg(objcg); + list_lru_del(&sbinfo->shrinklist, item, nid, memcg); + rcu_read_unlock(); +} + +static void shmem_unused_huge_add(struct inode *inode, struct folio *folio, + gfp_t gfp) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + int nid = folio_nid(folio); + struct obj_cgroup *objcg = NULL, *old_objcg = NULL; + + objcg = shmem_unused_huge_alloc_lru(sbinfo, folio, gfp); + if (IS_ERR(objcg)) + return; + + spin_lock(&info->lock); + if (!list_empty(&info->shrinklist)) { + /* isolated on scan list, let shrink handle it */ + if (is_shmem_unused_huge_isolated(info)) + goto unlock; + + if (info->shrinklist_nid == nid && + info->shrinklist_objcg == objcg) + goto unlock; + + shmem_unused_huge_lru_del(sbinfo, &info->shrinklist, + info->shrinklist_nid, + info->shrinklist_objcg); + old_objcg = shmem_get_and_clear_objcg(info); + } + + info->shrinklist_objcg = objcg; + info->shrinklist_nid = nid; + shmem_unused_huge_lru_add(sbinfo, &info->shrinklist, nid, objcg); + objcg = NULL; +unlock: + spin_unlock(&info->lock); + obj_cgroup_put(old_objcg); + obj_cgroup_put(objcg); +} + +static void shmem_unused_huge_del(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct obj_cgroup *objcg = NULL; + + spin_lock(&info->lock); + if (!list_empty(&info->shrinklist)) { + shmem_unused_huge_lru_del(sbinfo, &info->shrinklist, + info->shrinklist_nid, + info->shrinklist_objcg); + objcg = shmem_get_and_clear_objcg(info); + } + spin_unlock(&info->lock); + + obj_cgroup_put(objcg); +} + +struct shmem_unused_huge_scan { + struct list_head list; + struct shrink_control *sc; +}; + +static enum lru_status shmem_unused_huge_isolate(struct list_head *item, + struct list_lru_one *lru, + void *arg) +{ + struct shmem_unused_huge_scan *scan = arg; + struct shmem_inode_info *info; + struct inode *inode; + struct obj_cgroup *objcg = NULL; + + info = list_entry(item, struct shmem_inode_info, shrinklist); + + /* + * Use trylock to avoid ABBA deadlock: add/del path takes info->lock + * before the list_lru bucket lock, while here the order is reversed. + */ + if (!spin_trylock(&info->lock)) + return LRU_SKIP; + + /* pin the inode */ + inode = igrab(&info->vfs_inode); + /* inode is about to be evicted */ + if (!inode) { + list_lru_isolate(lru, item); + objcg = shmem_get_and_clear_objcg(info); + spin_unlock(&info->lock); + obj_cgroup_put(objcg); + return LRU_REMOVED; + } + + list_lru_isolate(lru, item); + objcg = shmem_get_and_clear_objcg(info); + set_shmem_unused_huge_isolated(info); + list_add_tail(&info->shrinklist, &scan->list); + spin_unlock(&info->lock); + obj_cgroup_put(objcg); + + return LRU_REMOVED; +} + +static bool is_shmem_unused_huge_match(struct folio *folio, + struct shrink_control *sc) +{ + struct mem_cgroup *memcg = NULL; + bool match; + + /* shmem quota reclaim has no NUMA node or memcg restriction */ + if (!sc) + return true; + + if (folio_nid(folio) != sc->nid) + return false; + + /* + * Only non-root memcg reclaim needs to match the folio charge against + * sc->memcg. Skip the folio memcg check for global shrinker reclaim and + * root memcg reclaim. + */ + if (!sc->memcg || mem_cgroup_is_root(sc->memcg)) + return true; + + memcg = get_mem_cgroup_from_folio(folio); + match = memcg == sc->memcg; + mem_cgroup_put(memcg); + + return match; +} + +static void shmem_unused_huge_drop(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + + spin_lock(&info->lock); + list_del_init(&info->shrinklist); + spin_unlock(&info->lock); +} + +static void shmem_unused_huge_requeue(struct inode *inode, struct folio *folio) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct obj_cgroup *objcg; + int nid = folio_nid(folio); + + objcg = shmem_unused_huge_alloc_lru(sbinfo, folio, GFP_NOWAIT); + if (IS_ERR(objcg)) { + shmem_unused_huge_drop(inode); + return; + } + + spin_lock(&info->lock); + /* Requeue the inode to shrinklist */ + list_del_init(&info->shrinklist); + shmem_unused_huge_lru_add(sbinfo, &info->shrinklist, nid, objcg); + info->shrinklist_objcg = objcg; + info->shrinklist_nid = nid; + spin_unlock(&info->lock); +} + static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, struct shrink_control *sc, unsigned long nr_to_free) { - LIST_HEAD(list), *pos, *next; + struct shmem_unused_huge_scan scan; struct inode *inode; struct shmem_inode_info *info; struct folio *folio; - unsigned long batch = sc ? sc->nr_to_scan : 128; + struct list_head *pos, *next; unsigned long split = 0, freed = 0; - if (list_empty(&sbinfo->shrinklist)) - return SHRINK_STOP; + INIT_LIST_HEAD(&scan.list); + scan.sc = sc; + if (sc) + list_lru_shrink_walk(&sbinfo->shrinklist, sc, + shmem_unused_huge_isolate, &scan); + else + list_lru_walk(&sbinfo->shrinklist, shmem_unused_huge_isolate, + &scan, 128); - spin_lock(&sbinfo->shrinklist_lock); - list_for_each_safe(pos, next, &sbinfo->shrinklist) { - info = list_entry(pos, struct shmem_inode_info, shrinklist); - - /* pin the inode */ - inode = igrab(&info->vfs_inode); - - /* inode is about to be evicted */ - if (!inode) { - list_del_init(&info->shrinklist); - goto next; - } - - list_move(&info->shrinklist, &list); -next: - sbinfo->shrinklist_len--; - if (!--batch) - break; - } - spin_unlock(&sbinfo->shrinklist_lock); - - list_for_each_safe(pos, next, &list) { - pgoff_t next, end; + list_for_each_safe(pos, next, &scan.list) { + pgoff_t folio_end, end; loff_t i_size; int ret; info = list_entry(pos, struct shmem_inode_info, shrinklist); inode = &info->vfs_inode; - if (nr_to_free && freed >= nr_to_free) - goto move_back; - i_size = i_size_read(inode); folio = filemap_get_entry(inode->i_mapping, i_size / PAGE_SIZE); if (!folio || xa_is_value(folio)) @@ -782,13 +989,19 @@ next: } /* Check if there is anything to gain from splitting */ - next = folio_next_index(folio); + folio_end = folio_next_index(folio); end = shmem_fallocend(inode, DIV_ROUND_UP(i_size, PAGE_SIZE)); - if (end <= folio->index || end >= next) { + if (end <= folio->index || end >= folio_end) { folio_put(folio); goto drop; } + if (!is_shmem_unused_huge_match(folio, scan.sc)) + goto move_back; + + if (nr_to_free && freed >= nr_to_free) + goto move_back; + /* * Move the inode on the list back to shrinklist if we failed * to lock the page at this time. @@ -796,35 +1009,30 @@ next: * Waiting for the lock may lead to deadlock in the * reclaim path. */ - if (!folio_trylock(folio)) { - folio_put(folio); + if (!folio_trylock(folio)) + goto move_back; + + if (!is_shmem_unused_huge_match(folio, scan.sc)) { + folio_unlock(folio); goto move_back; } ret = split_folio(folio); folio_unlock(folio); - folio_put(folio); /* If split failed move the inode on the list back to shrinklist */ if (ret) goto move_back; - freed += next - end; + freed += folio_end - end; split++; + folio_put(folio); drop: - list_del_init(&info->shrinklist); + shmem_unused_huge_drop(inode); goto put; move_back: - /* - * Make sure the inode is either on the global list or deleted - * from any local list before iput() since it could be deleted - * in another thread once we put the inode (then the local list - * is corrupted). - */ - spin_lock(&sbinfo->shrinklist_lock); - list_move(&info->shrinklist, &sbinfo->shrinklist); - sbinfo->shrinklist_len++; - spin_unlock(&sbinfo->shrinklist_lock); + shmem_unused_huge_requeue(inode, folio); + folio_put(folio); put: iput(inode); } @@ -837,7 +1045,7 @@ static long shmem_unused_huge_scan(struct super_block *sb, { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); - if (!READ_ONCE(sbinfo->shrinklist_len)) + if (!list_lru_shrink_count(&sbinfo->shrinklist, sc)) return SHRINK_STOP; return shmem_unused_huge_shrink(sbinfo, sc, 0); @@ -848,21 +1056,21 @@ static long shmem_unused_huge_count(struct super_block *sb, { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); - /* - * The per-superblock shrinklist is filesystem-global and does not - * honour sc->memcg, so it is only meaningful on the global (kswapd or - * root direct reclaim) shrink path. Skip the per-memcg iterations of - * shrink_slab_memcg() to avoid queueing duplicate global work. - */ - if (!mem_cgroup_shrink_is_root(sc)) - return 0; - - return READ_ONCE(sbinfo->shrinklist_len); + return list_lru_shrink_count(&sbinfo->shrinklist, sc); } #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ #define shmem_huge SHMEM_HUGE_DENY +static void shmem_unused_huge_add(struct inode *inode, struct folio *folio, + gfp_t gfp) +{ +} + +static void shmem_unused_huge_del(struct inode *inode) +{ +} + static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, struct shrink_control *sc, unsigned long nr_to_free) { @@ -976,6 +1184,7 @@ static long shmem_free_swap(struct address_space *mapping, pgoff_t index, pgoff_t end, void *radswap) { XA_STATE(xas, &mapping->i_pages, index); + const softleaf_t swp = radix_to_swp_entry(radswap); unsigned int nr_pages = 0; pgoff_t base; void *entry; @@ -992,8 +1201,9 @@ static long shmem_free_swap(struct address_space *mapping, } xas_unlock_irq(&xas); - if (nr_pages) - swap_put_entries_direct(radix_to_swp_entry(radswap), nr_pages); + /* A swapin-error marker holds no swap slot, so just drop it. */ + if (nr_pages && softleaf_is_swap(swp)) + swap_put_entries_direct(swp, nr_pages); return nr_pages; } @@ -1159,7 +1369,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, } folio_batch_remove_exceptionals(&fbatch); folio_batch_release(&fbatch); - cond_resched(); + cond_resched_tasks_rcu_qs(); } /* @@ -1200,7 +1410,7 @@ whole_folios: index = start; while (index < end) { - cond_resched(); + cond_resched_tasks_rcu_qs(); if (!find_get_entries(mapping, &index, end - 1, &fbatch, indices)) { @@ -1418,14 +1628,7 @@ static void shmem_evict_inode(struct inode *inode) inode->i_size = 0; mapping_set_exiting(inode->i_mapping); shmem_truncate_range(inode, 0, (loff_t)-1); - if (!list_empty(&info->shrinklist)) { - spin_lock(&sbinfo->shrinklist_lock); - if (!list_empty(&info->shrinklist)) { - list_del_init(&info->shrinklist); - sbinfo->shrinklist_len--; - } - spin_unlock(&sbinfo->shrinklist_lock); - } + shmem_unused_huge_del(inode); while (!list_empty(&info->swaplist)) { /* Wait while shmem_unuse() is scanning this inode... */ wait_var_event(&info->stop_eviction, @@ -1612,7 +1815,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); pgoff_t index; - int nr_pages; + int nr_pages, ret; bool split = false; if ((info->flags & SHMEM_F_LOCKED) || sbinfo->noswap) @@ -1693,7 +1896,8 @@ try_split: folio_mark_uptodate(folio); } - if (!folio_alloc_swap(folio)) { + ret = folio_alloc_swap(folio); + if (!ret) { bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages); int error; @@ -1746,7 +1950,7 @@ try_split: swap_cache_del_folio(folio); goto redirty; } - if (nr_pages > 1) + if (nr_pages > 1 && ret == -E2BIG) goto try_split; redirty: folio_mark_dirty(folio); @@ -2280,7 +2484,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, si = get_swap_device(index_entry); order = shmem_confirm_swap(mapping, index, index_entry); - if (unlikely(!si)) { + if (IS_ERR_OR_NULL(si)) { if (order < 0) return -EEXIST; else @@ -2539,27 +2743,6 @@ repeat: alloced: alloced = true; - if (folio_test_large(folio) && - DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < - folio_next_index(folio)) { - struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); - struct shmem_inode_info *info = SHMEM_I(inode); - /* - * Part of the large folio is beyond i_size: subject - * to shrink under memory pressure. - */ - spin_lock(&sbinfo->shrinklist_lock); - /* - * _careful to defend against unlocked access to - * ->shrink_list in shmem_unused_huge_shrink() - */ - if (list_empty_careful(&info->shrinklist)) { - list_add_tail(&info->shrinklist, - &sbinfo->shrinklist); - sbinfo->shrinklist_len++; - } - spin_unlock(&sbinfo->shrinklist_lock); - } if (sgp == SGP_WRITE) folio_set_referenced(folio); @@ -2589,6 +2772,20 @@ clear: error = -EINVAL; goto unlock; } + + /* + * Queue the inode on the shrink list only after all checks that might + * remove the folio have passed. Otherwise the inode could be left on + * the shrinker list with a stale folio. + */ + if (alloced && folio_test_large(folio) && + DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < folio_next_index(folio)) { + /* + * Part of the large folio is beyond i_size: subject + * to shrink under memory pressure. + */ + shmem_unused_huge_add(inode, folio, gfp); + } out: *foliop = folio; return 0; @@ -2789,7 +2986,7 @@ unsigned long shmem_get_unmapped_area(struct file *file, sb = file_inode(file)->i_sb; } else { /* - * Called directly from mm/mmap.c, or drivers/char/mem.c + * Called directly from mm/mmap.c, or mm/char-mem.c * for "/dev/zero", to create a shared anonymous object. */ if (IS_ERR(shm_mnt)) @@ -3065,6 +3262,10 @@ static struct inode *__shmem_get_inode(const struct mnt_idmap *idmap, if (info->fsflags) shmem_set_inode_flags(inode, info->fsflags, NULL); INIT_LIST_HEAD(&info->shrinklist); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + info->shrinklist_objcg = NULL; + info->shrinklist_nid = -1; +#endif INIT_LIST_HEAD(&info->swaplist); cache_no_acl(inode); if (sbinfo->noswap) @@ -4939,6 +5140,9 @@ static void shmem_put_super(struct super_block *sb) #endif free_percpu(sbinfo->ino_batch); percpu_counter_destroy(&sbinfo->used_blocks); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + list_lru_destroy(&sbinfo->shrinklist); +#endif mpol_put(sbinfo->mpol); #ifdef CONFIG_TMPFS_XATTR simple_xattr_cache_cleanup(&sbinfo->xa_cache); @@ -4956,6 +5160,9 @@ static const struct dentry_operations shmem_ci_dentry_ops = { static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + static struct lock_class_key shmem_unused_huge_lock_key; +#endif struct shmem_options *ctx = fc->fs_private; struct inode *inode; struct shmem_sb_info *sbinfo; @@ -5032,8 +5239,12 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) raw_spin_lock_init(&sbinfo->stat_lock); if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) goto failed; - spin_lock_init(&sbinfo->shrinklist_lock); - INIT_LIST_HEAD(&sbinfo->shrinklist); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (list_lru_init_memcg_key(&sbinfo->shrinklist, sb->s_shrink, + &shmem_unused_huge_lock_key)) + goto failed; +#endif sb->s_maxbytes = MAX_LFS_FILESIZE; sb->s_blocksize = PAGE_SIZE; @@ -5158,7 +5369,7 @@ static void __init shmem_destroy_inodecache(void) static int shmem_error_remove_folio(struct address_space *mapping, struct folio *folio) { - return 0; + return MF_DELAYED; } static const struct address_space_operations shmem_aops = { diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 5a2469fb1838..23b31bc4bf94 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -32,39 +32,23 @@ #include #include -#include "hugetlb_vmemmap.h" - /* * Flags for vmemmap_populate_range and friends. */ -/* Get a ref on the head page struct page, for ZONE_DEVICE compound pages */ -#define VMEMMAP_POPULATE_PAGEREF 0x0001 +/* Vmemmap population for ZONE_DEVICE compound pages */ +#define VMEMMAP_POPULATE_DAX 0x0001 #include "internal.h" #include "mm_init.h" #include "sparse.h" -/* - * Allocate a block of memory to be used to back the virtual memory map - * or to back the page tables that are used to create the mapping. - * Uses the main allocators if they are available, else bootmem. - */ - -static void * __ref __earlyonly_bootmem_alloc(int node, - unsigned long size, - unsigned long align, - unsigned long goal) -{ - return memmap_alloc(size, align, goal, node, false); -} - -void * __meminit vmemmap_alloc_block(unsigned long size, int node) +void __ref *vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN; int order = get_order(size); - static bool warned __meminitdata; + static bool warned; struct page *page; page = alloc_pages_node(node, gfp_mask, order); @@ -78,8 +62,7 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node) } return NULL; } else - return __earlyonly_bootmem_alloc(node, size, size, - __pa(MAX_DMA_ADDRESS)); + return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), node, false); } static void * __meminit altmap_alloc_block_buf(unsigned long size, @@ -148,37 +131,29 @@ void __meminit vmemmap_verify(pte_t *pte, int node, start, end - 1); } -static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, - struct vmem_altmap *altmap, - unsigned long ptpfn, unsigned long flags) +int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) { - pte_t *pte = pte_offset_kernel(pmd, addr); - if (pte_none(ptep_get(pte))) { - pte_t entry; - void *p; + const struct mem_section *ms = __pfn_to_section(pfn); + const int order = section_compound_order(ms); + const unsigned long pages_per_compound = 1UL << order; - if (ptpfn == (unsigned long)-1) { - p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); - if (!p) - return NULL; - ptpfn = PHYS_PFN(__pa(p)); - } else { - /* - * When a PTE/PMD entry is freed from the init_mm - * there's a free_pages() call to this page allocated - * above. Thus this get_page() is paired with the - * put_page_testzero() on the freeing path. - * This can only called by certain ZONE_DEVICE path, - * and through vmemmap_populate_compound_pages() when - * slab is available. - */ - if (flags & VMEMMAP_POPULATE_PAGEREF) - get_page(pfn_to_page(ptpfn)); - } - entry = pfn_pte(ptpfn, PAGE_KERNEL); - set_pte_at(&init_mm, addr, pte, entry); + VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); + VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); + + if (!section_vmemmap_optimizable(ms)) + return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); + + if (order < PFN_SECTION_SHIFT) { + VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound)); + return VMEMMAP_OPTIMIZATION_PAGES * nr_pages / pages_per_compound; } - return pte; + + VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)); + + if (IS_ALIGNED(pfn, pages_per_compound)) + return VMEMMAP_OPTIMIZATION_PAGES; + + return 0; } static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) @@ -192,6 +167,141 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) return p; } +#ifdef CONFIG_VMEMMAP_OPTIMIZATION +#define VMEMMAP_OPTIMIZATION_NR_ORDERS (MAX_FOLIO_ORDER - VMEMMAP_OPTIMIZATION_MIN_ORDER + 1) + +static __ref struct page **vmemmap_tails(struct zone *zone) +{ + const size_t size = array_size(VMEMMAP_OPTIMIZATION_NR_ORDERS, + sizeof(*zone->vmemmap_tails)); + struct page **pages = READ_ONCE(zone->vmemmap_tails); + + if (pages) + return pages; + + pages = slab_is_available() ? kzalloc_objs(*pages, VMEMMAP_OPTIMIZATION_NR_ORDERS) : + memblock_alloc(size, __alignof__(*pages)); + if (!pages) + return NULL; + + if (cmpxchg(&zone->vmemmap_tails, NULL, pages) != NULL) { + if (slab_is_available()) + kfree(pages); + else + memblock_free(pages, size); + pages = READ_ONCE(zone->vmemmap_tails); + } + + return pages; +} + +struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zone) +{ + const unsigned int idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; + struct page *page, **pages; + void *addr; + + if (WARN_ON_ONCE(idx >= VMEMMAP_OPTIMIZATION_NR_ORDERS)) + return NULL; + + pages = vmemmap_tails(zone); + if (!pages) + return NULL; + + page = READ_ONCE(pages[idx]); + if (page) + return page; + + addr = vmemmap_alloc_block(PAGE_SIZE, zone_to_nid(zone)); + if (!addr) + return NULL; + + for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) { + page = (struct page *)addr + i; + mm_zero_struct_page(page); + atomic_set(&page->_mapcount, -1); + set_page_node(page, zone_to_nid(zone)); + set_page_zone(page, zone_idx(zone)); + if (zone_is_zone_device(zone)) + __SetPageReserved(page); + prep_compound_tail(page, NULL, order); + } + + page = virt_to_page(addr); + if (cmpxchg(&pages[idx], NULL, page) != NULL) { + if (slab_is_available()) + __free_page(page); + else + memblock_free(addr, PAGE_SIZE); + page = READ_ONCE(pages[idx]); + } + + return page; +} +#endif + +static __meminit void *vmemmap_alloc_pte(unsigned long pfn, int node, + struct vmem_altmap *altmap, unsigned long flags) +{ + struct zone *zone; + struct page *page; + const unsigned int order = pfn_to_section_compound_order(pfn); + + /* + * Device DAX still relies on vmemmap_populate_compound_pages() for + * head/first-tail allocation and tail-page reuse. + */ + if (!vmemmap_optimizable_pfn(pfn) || flags & VMEMMAP_POPULATE_DAX) + return vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); + + zone = pfn_to_zone(pfn, node); + page = vmemmap_shared_tail_page(order, zone); + if (!page) + return NULL; + + return page_address(page); +} + +static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, + struct vmem_altmap *altmap, + unsigned long ptpfn, unsigned long flags) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + unsigned long pfn = page_to_pfn((struct page *)addr); + + if (pte_none(ptep_get(pte))) { + pte_t entry; + + if (ptpfn == (unsigned long)-1) { + void *p = vmemmap_alloc_pte(pfn, node, altmap, flags); + + if (!p) + return NULL; + ptpfn = PHYS_PFN(__pa(p)); + } else { + /* + * When a PTE/PMD entry is freed from the init_mm + * there's a free_pages() call to this page allocated + * above. Thus this try_get_page() is paired with the + * put_page_testzero() on the freeing path. + * This can only called by certain ZONE_DEVICE path, + * and through vmemmap_populate_compound_pages() when + * slab is available. + * + * Use try_get_page() to prevent the shared page refcount + * from overflowing. + */ + if ((flags & VMEMMAP_POPULATE_DAX) && + !try_get_page(pfn_to_page(ptpfn))) + return NULL; + } + entry = pfn_pte(ptpfn, PAGE_KERNEL); + set_pte_at(&init_mm, addr, pte, entry); + } else if (WARN_ON_ONCE(vmemmap_optimizable_pfn(pfn))) + return NULL; + return pte; +} + static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) { pmd_t *pmd = pmd_offset(pud, addr); @@ -322,69 +432,6 @@ void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end, } } -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP -static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) -{ - struct page *p, *tail; - unsigned int idx; - int node = zone_to_nid(zone); - - if (WARN_ON_ONCE(order < VMEMMAP_TAIL_MIN_ORDER)) - return NULL; - if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER)) - return NULL; - - idx = order - VMEMMAP_TAIL_MIN_ORDER; - tail = zone->vmemmap_tails[idx]; - if (tail) - return tail; - - /* - * Only allocate the page, but do not initialize it. - * - * Any initialization done here will be overwritten by memmap_init(). - * - * hugetlb_bootmem_struct_page_init() will take care of initialization - * after memmap_init(). - */ - - p = vmemmap_alloc_block_zero(PAGE_SIZE, node); - if (!p) - return NULL; - - tail = virt_to_page(p); - zone->vmemmap_tails[idx] = tail; - - return tail; -} - -int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end, - unsigned int order, struct zone *zone, - unsigned long headsize) -{ - unsigned long maddr; - struct page *tail; - pte_t *pte; - int node = zone_to_nid(zone); - - tail = vmemmap_get_tail(order, zone); - if (!tail) - return -ENOMEM; - - for (maddr = addr; maddr < addr + headsize; maddr += PAGE_SIZE) { - pte = vmemmap_populate_address(maddr, node, NULL, -1, 0); - if (!pte) - return -ENOMEM; - } - - /* - * Reuse the last page struct page mapped above for the rest. - */ - return vmemmap_populate_range(maddr, end, node, NULL, - page_to_pfn(tail), 0); -} -#endif - void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node, unsigned long addr, unsigned long next) { @@ -412,6 +459,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, pmd_t *pmd; for (addr = start; addr < end; addr = next) { + unsigned long pfn = page_to_pfn((struct page *)addr); + const struct mem_section *ms = __pfn_to_section(pfn); + next = pmd_addr_end(addr, end); pgd = vmemmap_pgd_populate(addr, node); @@ -427,7 +477,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, return -ENOMEM; pmd = pmd_offset(pud, addr); - if (pmd_none(pmdp_get(pmd))) { + if (pmd_none(pmdp_get(pmd)) && !section_vmemmap_optimizable(ms)) { void *p; p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap); @@ -445,8 +495,11 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, */ return -ENOMEM; } - } else if (vmemmap_check_pmd(pmd, node, addr, next)) + } else if (vmemmap_check_pmd(pmd, node, addr, next)) { + if (WARN_ON_ONCE(section_vmemmap_optimizable(ms))) + return -EOPNOTSUPP; continue; + } if (vmemmap_populate_basepages(addr, next, node, altmap)) return -ENOMEM; } @@ -474,69 +527,42 @@ static bool __meminit reuse_compound_section(unsigned long start_pfn, return !IS_ALIGNED(offset, nr_pages) && nr_pages > PAGES_PER_SUBSECTION; } -static pte_t * __meminit compound_section_tail_page(unsigned long addr) -{ - pte_t *pte; - - addr -= PAGE_SIZE; - - /* - * Assuming sections are populated sequentially, the previous section's - * page data can be reused. - */ - pte = pte_offset_kernel(pmd_off_k(addr), addr); - if (!pte) - return NULL; - - return pte; -} - static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, unsigned long start, unsigned long end, int node, struct dev_pagemap *pgmap) { + const unsigned long flags = VMEMMAP_POPULATE_DAX; + const unsigned int order = pfn_to_section_compound_order(start_pfn); unsigned long size, addr; pte_t *pte; + struct page *page; int rc; - if (reuse_compound_section(start_pfn, pgmap)) { - pte = compound_section_tail_page(start); - if (!pte) - return -ENOMEM; + page = vmemmap_shared_tail_page(order, device_zone(node)); + if (!page) + return -ENOMEM; - /* - * Reuse the page that was populated in the prior iteration - * with just tail struct pages. - */ + if (reuse_compound_section(start_pfn, pgmap)) return vmemmap_populate_range(start, end, node, NULL, - pte_pfn(ptep_get(pte)), - VMEMMAP_POPULATE_PAGEREF); - } + page_to_pfn(page), flags); - size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page)); + size = min(end - start, (1UL << order) * sizeof(struct page)); for (addr = start; addr < end; addr += size) { unsigned long next, last = addr + size; /* Populate the head page vmemmap page */ - pte = vmemmap_populate_address(addr, node, NULL, -1, 0); - if (!pte) - return -ENOMEM; - - /* Populate the tail pages vmemmap page */ - next = addr + PAGE_SIZE; - pte = vmemmap_populate_address(next, node, NULL, -1, 0); + pte = vmemmap_populate_address(addr, node, NULL, -1, flags); if (!pte) return -ENOMEM; /* - * Reuse the previous page for the rest of tail pages + * Reuse the shared page for the rest of tail pages * See layout diagram in Documentation/mm/vmemmap_dedup.rst */ - next += PAGE_SIZE; + next = addr + PAGE_SIZE; rc = vmemmap_populate_range(next, last, node, NULL, - pte_pfn(ptep_get(pte)), - VMEMMAP_POPULATE_PAGEREF); + page_to_pfn(page), flags); if (rc) return -ENOMEM; } @@ -558,7 +584,7 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn, !IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION))) return NULL; - if (vmemmap_can_optimize(altmap, pgmap)) + if (pgmap && section_vmemmap_optimizable(__pfn_to_section(pfn))) r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap); else r = vmemmap_populate(start, end, nid, altmap); @@ -571,20 +597,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn, return pfn_to_page(pfn); } -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -/* - * This is called just before initializing sections for a NUMA node. - * Any special initialization that needs to be done before the - * generic initialization can be done from here. Sections that - * are initialized in hooks called from here will be skipped by - * the generic initialization. - */ -void __init sparse_vmemmap_init_nid_early(int nid) -{ - hugetlb_vmemmap_init_early(nid); -} -#endif - static void subsection_mask_set(unsigned long *map, unsigned long pfn, unsigned long nr_pages) { @@ -654,31 +666,6 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn) } } -static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, - struct vmem_altmap *altmap, struct dev_pagemap *pgmap) -{ - const unsigned int order = pgmap ? pgmap->vmemmap_shift : 0; - const unsigned long pages_per_compound = 1UL << order; - - VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); - VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); - - if (!vmemmap_can_optimize(altmap, pgmap)) - return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); - - if (order < PFN_SECTION_SHIFT) { - VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound)); - return VMEMMAP_RESERVE_NR * nr_pages / pages_per_compound; - } - - VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)); - - if (IS_ALIGNED(pfn, pages_per_compound)) - return VMEMMAP_RESERVE_NR; - - return 0; -} - static struct page * __meminit populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) @@ -686,7 +673,7 @@ static struct page * __meminit populate_section_memmap(unsigned long pfn, struct page *page = __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); - memmap_pages_add(section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap)); + memmap_pages_add(section_nr_vmemmap_pages(pfn, nr_pages)); return page; } @@ -697,7 +684,7 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, unsigned long start = (unsigned long) pfn_to_page(pfn); unsigned long end = start + nr_pages * sizeof(struct page); - memmap_pages_add(-section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap)); + memmap_pages_add(-section_nr_vmemmap_pages(pfn, nr_pages)); vmemmap_free(start, end, altmap); } @@ -707,8 +694,7 @@ static void free_map_bootmem(struct page *memmap) unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); unsigned long pfn = page_to_pfn(memmap); - memmap_boot_pages_add(-section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION, - NULL, NULL)); + memmap_boot_pages_add(-section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION)); vmemmap_free(start, end, NULL); } @@ -816,8 +802,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, else if (memmap) free_map_bootmem(memmap); - if (empty) + if (empty) { ms->section_mem_map = (unsigned long)NULL; + section_set_compound_order(ms, 0); + } } static struct page * __meminit section_activate(int nid, unsigned long pfn, @@ -827,8 +815,13 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, struct mem_section *ms = __pfn_to_section(pfn); struct mem_section_usage *usage = NULL; struct page *memmap; + unsigned int order; int rc; + order = vmemmap_can_optimize(altmap, pgmap) ? pgmap->vmemmap_shift : 0; + if (nr_pages < PAGES_PER_SECTION && section_compound_order(ms)) + return ERR_PTR(-EOPNOTSUPP); + if (!ms->usage) { usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); if (!usage) @@ -854,6 +847,7 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, if (nr_pages < PAGES_PER_SECTION && early_section(ms)) return pfn_to_page(pfn); + section_set_compound_order_range(pfn, nr_pages, order); memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); if (!memmap) { section_deactivate(pfn, nr_pages, altmap, pgmap); @@ -899,13 +893,16 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn, if (IS_ERR(memmap)) return PTR_ERR(memmap); - /* - * Poison uninitialized struct pages in order to catch invalid flags - * combinations. - */ - page_init_poison(memmap, sizeof(struct page) * nr_pages); - ms = __nr_to_section(section_nr); + /* + * Poison uninitialized struct pages to catch invalid flag combinations. + * + * Tail struct pages in a vmemmap-optimized section are initialized and + * shared during vmemmap population, so they must not be overwritten here. + */ + if (!section_vmemmap_optimizable(ms)) + page_init_poison(memmap, sizeof(struct page) * nr_pages); + __section_mark_present(ms, section_nr); /* Align memmap to section boundary in the subsection case */ diff --git a/mm/sparse.c b/mm/sparse.c index 7c15406e77f5..b75921c622ed 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -191,12 +191,7 @@ static void __init memory_present(int nid, unsigned long start, unsigned long en } } -/* - * Mark all memblocks as present using memory_present(). - * This is a convenience function that is useful to mark all of the systems - * memory as present during initialization. - */ -static void __init memblocks_present(void) +void __init sparse_sections_init(void) { unsigned long start, end; int i, nid; @@ -213,23 +208,12 @@ static void __init memblocks_present(void) memory_present(nid, start, end); } -#ifdef CONFIG_SPARSEMEM_VMEMMAP -unsigned long __init section_map_size(void) -{ - return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); -} - -#else -unsigned long __init section_map_size(void) -{ - return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); -} - +#ifndef CONFIG_SPARSEMEM_VMEMMAP struct page __init *__populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - unsigned long size = section_map_size(); + const unsigned long size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); } @@ -239,42 +223,6 @@ void __weak __meminit vmemmap_populate_print_last(void) { } -static void *sparse_usagebuf __initdata; -static void *sparse_usagebuf_end __initdata; - -/* - * Helper function that is used for generic section initialization, and - * can also be used by any hooks added above. - */ -void __init sparse_init_early_section(int nid, struct page *map, - unsigned long pnum, unsigned long flags) -{ - BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end); - sparse_init_one_section(__nr_to_section(pnum), pnum, map, - sparse_usagebuf, SECTION_IS_EARLY | flags); - sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size(); -} - -static int __init sparse_usage_init(int nid, unsigned long map_count) -{ - unsigned long size; - - size = mem_section_usage_size() * map_count; - sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid); - if (!sparse_usagebuf) { - sparse_usagebuf_end = NULL; - return -ENOMEM; - } - - sparse_usagebuf_end = sparse_usagebuf + size; - return 0; -} - -static void __init sparse_usage_fini(void) -{ - sparse_usagebuf = sparse_usagebuf_end = NULL; -} - /* * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) * And number of present sections in this node is map_count. @@ -284,33 +232,29 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, unsigned long map_count) { unsigned long pnum; + struct mem_section_usage *usage; - if (sparse_usage_init(nid, map_count)) + usage = memblock_alloc_node(map_count * mem_section_usage_size(), + SMP_CACHE_BYTES, nid); + if (!usage) panic("Failed to allocate usemap for node %d\n", nid); - sparse_vmemmap_init_nid_early(nid); - for_each_present_section_nr(pnum_begin, pnum) { - struct mem_section *ms; unsigned long pfn = section_nr_to_pfn(pnum); + struct page *map; if (pnum >= pnum_end) break; - ms = __nr_to_section(pnum); - if (!preinited_vmemmap_section(ms)) { - struct page *map; - - map = __populate_section_memmap(pfn, PAGES_PER_SECTION, - nid, NULL, NULL); - if (!map) - panic("Failed to allocate memmap for section %lu\n", pnum); - memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), - PAGE_SIZE)); - sparse_init_early_section(nid, map, pnum, 0); - } + map = __populate_section_memmap(pfn, PAGES_PER_SECTION, + nid, NULL, NULL); + if (!map) + panic("Failed to allocate memmap for section %lu\n", pnum); + memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION)); + sparse_init_one_section(__nr_to_section(pnum), pnum, map, usage, + SECTION_IS_EARLY); + usage = (void *)usage + mem_section_usage_size(); } - sparse_usage_fini(); } /* @@ -322,10 +266,6 @@ void __init sparse_init(void) unsigned long pnum_end, pnum_begin, map_count = 1; int nid_begin; - /* see include/linux/mmzone.h 'struct mem_section' definition */ - BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); - memblocks_present(); - if (compound_info_has_mask()) { VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), MAX_FOLIO_VMEMMAP_ALIGN)); diff --git a/mm/sparse.h b/mm/sparse.h index 3b744667a7e6..530692cdd516 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -9,12 +9,14 @@ #define __MM_SPARSE_H #include +#include /* * mm/sparse.c */ #ifdef CONFIG_SPARSEMEM void sparse_init(void); +void sparse_sections_init(void); int sparse_index_init(unsigned long section_nr, int nid); static inline void sparse_init_one_section(struct mem_section *ms, @@ -53,8 +55,14 @@ static inline size_t mem_section_usage_size(void) return struct_size_t(struct mem_section_usage, pageblock_flags, BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS)); } + +static inline bool section_vmemmap_optimizable(const struct mem_section *ms) +{ + return vmemmap_optimizable_order(section_compound_order(ms)); +} #else static inline void sparse_init(void) {} +static inline void sparse_sections_init(void) {} #endif /* CONFIG_SPARSEMEM */ /* @@ -62,8 +70,13 @@ static inline void sparse_init(void) {} */ #ifdef CONFIG_SPARSEMEM_VMEMMAP void sparse_init_subsection_map(void); +int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages); #else static inline void sparse_init_subsection_map(void) {} +static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) +{ + return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); +} #endif /* CONFIG_SPARSEMEM_VMEMMAP */ #endif /* __MM_SPARSE_H */ diff --git a/mm/swap.h b/mm/swap.h index 90a551a88df6..d5bf21f517dc 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -82,7 +82,7 @@ enum swap_cluster_flags { extern int vm_swappiness; -static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) +static inline int mem_cgroup_swappiness(const struct mem_cgroup *memcg) { #ifdef CONFIG_MEMCG_V1 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && @@ -258,7 +258,7 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio); void swap_read_submit(struct swap_io_ctx *ctx); void swap_write_submit(struct swap_io_ctx *ctx); int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); -void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio); +void __swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); /* linux/mm/swap_state.c */ extern struct address_space swap_space __read_mostly; @@ -312,19 +312,19 @@ bool swap_cache_has_folio(swp_entry_t entry); struct folio *swap_cache_get_folio(swp_entry_t entry); void *swap_cache_get_shadow(swp_entry_t entry); void swap_cache_del_folio(struct folio *folio); -struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, - unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx); +struct folio *__swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, + unsigned long orders, struct vm_fault *vmf, + struct mempolicy *mpol, pgoff_t ilx); /* Below helpers require the caller to lock and pass in the swap cluster. */ void __swap_cache_add_folio(struct swap_cluster_info *ci, struct folio *folio, swp_entry_t entry); void __swap_cache_del_folio(struct swap_cluster_info *ci, - struct folio *folio, swp_entry_t entry, void *shadow); + struct folio *folio, swp_entry_t entry, void *shadow, + bool swapout); void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new); void show_swap_cache_info(void); -void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr); struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, @@ -453,7 +453,8 @@ static inline void swap_cache_del_folio(struct folio *folio) } static inline void __swap_cache_del_folio(struct swap_cluster_info *ci, - struct folio *folio, swp_entry_t entry, void *shadow) + struct folio *folio, swp_entry_t entry, void *shadow, + bool swapout) { } diff --git a/mm/swap_state.c b/mm/swap_state.c index f3961fdd857d..2475ba29126d 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -306,21 +306,28 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci, * @folio: The folio. * @entry: The first swap entry that the folio corresponds to. * @shadow: shadow value to be filled in the swap cache. + * @swapout: whether this folio is being reclaimed after swapout. * * Removes a folio from the swap cache and fills a shadow in place. * This won't put the folio's refcount. The caller has to do that. * * Context: Caller must ensure the folio is locked and in the swap cache * using the index of @entry, and lock the cluster that holds the entries. + * If @swapout is set, the folio should be in reclaim path and IRQs + * should be disabled. */ void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio, - swp_entry_t entry, void *shadow) + swp_entry_t entry, void *shadow, bool swapout) { unsigned long nr_pages = folio_nr_pages(folio); - __swap_cache_do_del_folio(ci, folio, entry, shadow); node_stat_mod_folio(folio, NR_FILE_PAGES, -nr_pages); lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr_pages); + + if (swapout) + __memcg1_swapout(folio, ci); + + __swap_cache_do_del_folio(ci, folio, entry, shadow); } /** @@ -339,7 +346,7 @@ void swap_cache_del_folio(struct folio *folio) swp_entry_t entry = folio->swap; ci = swap_cluster_lock(__swap_entry_to_info(entry), swp_offset(entry)); - __swap_cache_del_folio(ci, folio, entry, NULL); + __swap_cache_del_folio(ci, folio, entry, NULL, false); swap_cluster_unlock(ci); folio_ref_sub(folio, folio_nr_pages(folio)); @@ -389,8 +396,9 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, folio_order(old) != folio_order(new)) { ci_off = swp_cluster_offset(old->swap); ci_end = ci_off + folio_nr_pages(old); - while (ci_off++ < ci_end) + do { WARN_ON_ONCE(swp_tb_to_folio(__swap_table_get(ci, ci_off)) != old); + } while (++ci_off < ci_end); } } @@ -489,13 +497,11 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages); lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages); - /* Caller will initiate read into locked new_folio */ - folio_add_lru(folio); return folio; } /** - * swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache. + * __swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache. * @targ_entry: swap entry indicating the target slot * @gfp: memory allocation flags * @orders: allocation orders, must be non zero @@ -507,13 +513,17 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, * doing IO (e.g. swap in or zswap writeback). The swap slot indicated by * @targ_entry must have a non-zero swap count (swapped out). * + * The returned folio is locked and is NOT on the LRU. The caller must either + * add it to the LRU with folio_add_lru() so page reclaim can find it, or free + * it directly once done; a folio left off the LRU is unreclaimable and leaks. + * * Context: Caller must protect the swap device with reference count or locks. * Return: Returns the folio if allocation succeeded and folio is in the swap * cache. Returns error code if failed due to race, OOM or invalid arguments. */ -struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, - unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx) +struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, + unsigned long orders, struct vm_fault *vmf, + struct mempolicy *mpol, pgoff_t ilx) { int order, err; struct folio *ret; @@ -541,6 +551,48 @@ struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, return ret; } +/** + * swap_writeback_dropbehind_folio - drop a dropbehind swap cache folio + * @folio: the off-LRU folio whose writeback has completed + * + * Context: task context, with the reference taken by folio_end_writeback() + * donated to us. + */ +void swap_writeback_dropbehind_folio(struct folio *folio) +{ + struct mem_cgroup *memcg; + + folio_lock(folio); + + /* The folio was allocated off the LRU and nothing re-adds it here. */ + VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); + + rcu_read_lock(); + memcg = folio_memcg(folio); + if (!mem_cgroup_tryget(memcg)) + memcg = NULL; + rcu_read_unlock(); + + /* + * Gate remove_mapping_set_shadow() on folio_test_swapcache(): a racing + * swapin may have freed the swap slot (folio_free_swap()) and dropped the + * folio from the cache, and it must not run on a non-swapcache folio (it + * would trip __remove_mapping()'s mapping == folio_mapping() check). + */ + if (!folio_test_swapcache(folio) || folio_test_writeback(folio) || + !remove_mapping_set_shadow(swap_address_space(folio->swap), folio, + memcg)) { + /* Raced: the folio is now owned by the swapin; put it back. */ + folio_clear_dropbehind(folio); + folio_add_lru(folio); + } + + mem_cgroup_put(memcg); + + folio_unlock(folio); + folio_put(folio); +} + /* * If we are the only user, then try to free up the swap cache. * @@ -649,12 +701,13 @@ static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx, folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR_OR_NULL(folio)) return NULL; + folio_add_lru(folio); swap_read_folio(ctx, folio); if (readahead) { folio_set_readahead(folio); @@ -690,12 +743,13 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR(folio)) return folio; + folio_add_lru(folio); swap_read_folio(&ctx, folio); swap_read_submit(&ctx); return folio; @@ -716,7 +770,7 @@ struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, struct folio *folio; si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return NULL; mpol = get_vma_policy(vma, addr, 0, &ilx); @@ -952,7 +1006,7 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, */ if (swp_type(entry) != swp_type(targ_entry)) { si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) continue; } folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx, diff --git a/mm/swapfile.c b/mm/swapfile.c index 601979b97f95..16ceaa28db3a 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -370,8 +370,6 @@ static void discard_swap_cluster(struct swap_info_struct *si, } } -#define LATENCY_LIMIT 256 - static inline bool cluster_is_empty(struct swap_cluster_info *info) { return info->count == 0; @@ -418,6 +416,17 @@ static void swap_cluster_free_table_folio_rcu_cb(struct rcu_head *head) folio_put(folio); } +static void swap_cluster_free_count_table(struct swap_table *table) +{ + if (!SWP_TABLE_USE_PAGE) { + kmem_cache_free(swap_table_cachep, table); + return; + } + + call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head), + swap_cluster_free_table_folio_rcu_cb); +} + static void swap_cluster_free_table(struct swap_cluster_info *ci) { struct swap_table *table; @@ -437,13 +446,7 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci) return; rcu_assign_pointer(ci->table, NULL); - if (!SWP_TABLE_USE_PAGE) { - kmem_cache_free(swap_table_cachep, table); - return; - } - - call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head), - swap_cluster_free_table_folio_rcu_cb); + swap_cluster_free_count_table(table); } static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp) @@ -466,14 +469,12 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp) if (!table) return -ENOMEM; - rcu_assign_pointer(ci->table, table); - #ifdef CONFIG_MEMCG if (!mem_cgroup_disabled()) { VM_WARN_ON_ONCE(ci->memcg_table); ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp); if (!ci->memcg_table) { - swap_cluster_free_table(ci); + swap_cluster_free_count_table(table); return -ENOMEM; } } @@ -484,9 +485,16 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp) ci->zero_bitmap = bitmap_zalloc(SWAPFILE_CLUSTER, gfp); if (!ci->zero_bitmap) { swap_cluster_free_table(ci); + swap_cluster_free_count_table(table); return -ENOMEM; } #endif + + /* + * Make tables visible to cluster_is_usable() after everything is + * ready. + */ + rcu_assign_pointer(ci->table, table); return 0; } @@ -1318,10 +1326,8 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset, { unsigned long end = offset + nr_entries - 1; void (*swap_slot_free_notify)(struct block_device *, unsigned long); - unsigned int i; - for (i = 0; i < nr_entries; i++) - zswap_invalidate(swp_entry(si->type, offset + i)); + zswap_invalidate(si->type, offset, nr_entries); if (si->flags & SWP_BLKDEV) swap_slot_free_notify = @@ -1504,7 +1510,7 @@ int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp) unsigned long offset = swp_offset(entry); si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return 0; ci = __swap_offset_to_cluster(si, offset); @@ -1517,20 +1523,17 @@ int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp) static void swap_extend_table_try_free(struct swap_cluster_info *ci) { unsigned long i; - bool can_free = true; if (!ci->extend_table) return; for (i = 0; i < SWAPFILE_CLUSTER; i++) { if (ci->extend_table[i]) - can_free = false; + return; } - if (can_free) { - kfree(ci->extend_table); - ci->extend_table = NULL; - } + kfree(ci->extend_table); + ci->extend_table = NULL; } /* Decrease the swap count of one slot, without freeing it */ @@ -1722,7 +1725,6 @@ restart: failed: while (ci_off-- > ci_start) __swap_cluster_put_entry(ci, ci_off); - swap_extend_table_try_free(ci); swap_cluster_unlock(ci); return err; } @@ -1735,7 +1737,9 @@ failed: * swap cache. * * Context: Caller needs to hold the folio lock. - * Return: Whether the folio was added to the swap cache. + * Return: %0 on success, %-E2BIG if splitting the folio might allow swapout, + * %-ENOSPC if no global swap space is available, or %-ENOMEM if splitting + * would not help. */ int folio_alloc_swap(struct folio *folio) { @@ -1747,11 +1751,11 @@ int folio_alloc_swap(struct folio *folio) if (order) { /* - * Reject large allocation when THP_SWAP is disabled, - * the caller should split the folio and try again. + * Reject large allocation when THP_SWAP is disabled. Check below + * whether splitting and retrying can make progress. */ if (!IS_ENABLED(CONFIG_THP_SWAP)) - return -EAGAIN; + goto failed; /* * Allocation size should never exceed cluster size @@ -1759,7 +1763,7 @@ int folio_alloc_swap(struct folio *folio) */ if (size > SWAPFILE_CLUSTER) { VM_WARN_ON_ONCE(1); - return -EINVAL; + goto failed; } } @@ -1775,13 +1779,23 @@ again: } /* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */ - if (unlikely(mem_cgroup_try_charge_swap(folio))) + if (unlikely(mem_cgroup_try_charge_swap(folio))) { swap_cache_del_folio(folio); + goto failed; + } if (unlikely(!folio_test_swapcache(folio))) - return -ENOMEM; + goto failed; return 0; + +failed: + if (get_nr_swap_pages() <= 0) + return -ENOSPC; + if (mem_cgroup_get_folio_swap_margin(folio) <= 0) + return -ENOMEM; + + return order ? -E2BIG : -ENOMEM; } /** @@ -1810,7 +1824,7 @@ int folio_dup_swap(struct folio *folio, struct page *page) VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); if (page) { - entry.val += folio_page_idx(folio, page); + entry = folio_page_swap_entry(folio, page); nr_pages = 1; } @@ -1837,7 +1851,7 @@ void folio_put_swap(struct folio *folio, struct page *page) VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); if (page) { - entry.val += folio_page_idx(folio, page); + entry = folio_page_swap_entry(folio, page); nr_pages = 1; } @@ -1859,7 +1873,10 @@ void folio_put_swap(struct folio *folio, struct page *page) * Check whether swap entry is valid in the swap device. If so, * return pointer to swap_info_struct, and keep the swap entry valid * via preventing the swap device from being swapoff, until - * put_swap_device() is called. Otherwise return NULL. + * put_swap_device() is called. Return NULL for an empty entry or a + * device that is going away, and ERR_PTR(-EIO) if the entry's type + * names no swap device or its offset is past the end of one. These EIOs + * are preceded by pr_err(). * * Notice that swapoff or swapoff+swapon can still happen before the * percpu_ref_tryget_live() in get_swap_device() or after the @@ -1870,7 +1887,7 @@ void folio_put_swap(struct folio *folio, struct page *page) * CPU1 CPU2 * do_swap_page() * ... swapoff+swapon - * swap_cache_alloc_folio() + * __swap_cache_alloc_folio() * // check swap_map * // verify PTE not changed * @@ -1900,12 +1917,14 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry) return si; bad_nofile: pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val); + return ERR_PTR(-EIO); + out: return NULL; put_out: pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val); percpu_ref_put(&si->users); - return NULL; + return ERR_PTR(-EIO); } /* @@ -2001,7 +2020,7 @@ int swp_swapcount(swp_entry_t entry) int count; si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return 0; ci = swap_cluster_lock(si, swp_offset(entry)); @@ -2127,7 +2146,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) struct swap_info_struct *si; si = get_swap_device(entry); - if (WARN_ON_ONCE(!si)) + if (WARN_ON_ONCE(IS_ERR_OR_NULL(si))) return; if (WARN_ON_ONCE(end_offset > si->max)) goto out; @@ -2709,7 +2728,9 @@ unlock: static unsigned int find_next_to_unuse(struct swap_info_struct *si, unsigned int prev) { - unsigned int i; + struct swap_cluster_info *ci; + unsigned long i, end; + unsigned int ci_off; unsigned long swp_tb; /* @@ -2718,19 +2739,36 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si, * hits are okay, and sys_swapoff() has already prevented new * allocations from this area (while holding swap_lock). */ - for (i = prev + 1; i < si->max; i++) { - swp_tb = swap_table_get(__swap_offset_to_cluster(si, i), - i % SWAPFILE_CLUSTER); - if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb)) - break; - if ((i % LATENCY_LIMIT) == 0) + i = prev + 1; + while (i < si->max) { + ci = __swap_offset_to_cluster(si, i); + end = min_t(unsigned long, + ALIGN_DOWN(i, SWAPFILE_CLUSTER) + SWAPFILE_CLUSTER, + si->max); + + /* + * An empty cluster has no slot in use, so skip it whole. + * A slot is uncounted only after its folio left the swap + * cache, so there is nothing here for try_to_unuse() to act on. + * Count only drops here, so a READ_ONCE() without ci->lock is + * enough, unlike in every other cluster_is_empty() caller. + */ + if (!READ_ONCE(ci->count)) { + i = end; cond_resched(); + continue; + } + + ci_off = i % SWAPFILE_CLUSTER; + for (; i < end; ci_off++, i++) { + swp_tb = swap_table_get(ci, ci_off); + if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb)) + return i; + } + cond_resched(); } - if (i == si->max) - i = 0; - - return i; + return 0; } static int try_to_unuse(unsigned int type) @@ -2928,7 +2966,7 @@ EXPORT_SYMBOL_GPL(add_swap_extent); /* * A `swap extent' is a simple thing which maps a contiguous range of pages * onto a contiguous range of disk blocks. A rbtree of swap extents is - * built at swapon time and is then used at swap_writepage/swap_read_folio + * built at swapon time and is then used at swap_writeout/swap_read_folio * time for locating where on disk a page belongs. * * If the swapfile is an S_ISBLK block device, a single extent is installed. @@ -3721,11 +3759,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) maxpages = si->max; - /* Set up the swap cluster info */ - error = setup_swap_clusters_info(si, swap_header, maxpages); - if (error) - goto bad_swap_unlock_inode; - if (si->bdev && bdev_stable_writes(si->bdev)) si->flags |= SWP_STABLE_WRITES; @@ -3739,6 +3772,14 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) inced_nr_rotate_swap = true; } + /* + * Set up the swap cluster info after SWP_ flags handling as + * setup_swap_clusters_info() checks SWP_SOLIDSTATE. + */ + error = setup_swap_clusters_info(si, swap_header, maxpages); + if (error) + goto bad_swap_unlock_inode; + if ((swap_flags & SWAP_FLAG_DISCARD) && si->bdev && bdev_max_discard_sectors(si->bdev)) { /* diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 6a033df14027..938f20e9a78f 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -122,7 +122,6 @@ struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, return vma; } -#ifdef CONFIG_PER_VMA_LOCK /* * uffd_lock_vma() - Lookup and lock vma corresponding to @address. * @mm: mm to search vma in. @@ -130,8 +129,10 @@ struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, * * Should be called without holding mmap_lock. * - * Return: A locked vma containing @address, -ENOENT if no vma is found, or - * -ENOMEM if anon_vma couldn't be allocated. + * Return: A locked vma containing @address, -ENOENT if no vma is found, + * -ENOMEM if anon_vma couldn't be allocated, or -EAGAIN if vma refcount + * overflow happened due to high number of readers and the caller should + * retry later. */ static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm, unsigned long address) @@ -182,34 +183,6 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma) vma_end_read(vma); } -#else - -static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, - unsigned long dst_start, - unsigned long len) -{ - struct vm_area_struct *dst_vma; - - mmap_read_lock(dst_mm); - dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); - if (IS_ERR(dst_vma)) - goto out_unlock; - - if (validate_dst_vma(dst_vma, dst_start + len)) - return dst_vma; - - dst_vma = ERR_PTR(-ENOENT); -out_unlock: - mmap_read_unlock(dst_mm); - return dst_vma; -} - -static void uffd_mfill_unlock(struct vm_area_struct *vma) -{ - mmap_read_unlock(vma->vm_mm); -} -#endif - static void mfill_put_vma(struct mfill_state *state) { if (!state->vma) @@ -1700,7 +1673,7 @@ retry: } si = get_swap_device(entry); - if (unlikely(!si)) { + if (IS_ERR_OR_NULL(si)) { ret = -EAGAIN; goto out; } @@ -1757,7 +1730,7 @@ out: if (dst_pte) pte_unmap(dst_pte); mmu_notifier_invalidate_range_end(&range); - if (si) + if (!IS_ERR_OR_NULL(si)) put_swap_device(si); return ret; @@ -1850,7 +1823,6 @@ out_success: return 0; } -#ifdef CONFIG_PER_VMA_LOCK static int uffd_move_lock(struct mm_struct *mm, unsigned long dst_start, unsigned long src_start, @@ -1925,31 +1897,6 @@ static void uffd_move_unlock(struct vm_area_struct *dst_vma, vma_end_read(dst_vma); } -#else - -static int uffd_move_lock(struct mm_struct *mm, - unsigned long dst_start, - unsigned long src_start, - struct vm_area_struct **dst_vmap, - struct vm_area_struct **src_vmap) -{ - int err; - - mmap_read_lock(mm); - err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); - if (err) - mmap_read_unlock(mm); - return err; -} - -static void uffd_move_unlock(struct vm_area_struct *dst_vma, - struct vm_area_struct *src_vma) -{ - mmap_assert_locked(src_vma->vm_mm); - mmap_read_unlock(dst_vma->vm_mm); -} -#endif - /** * move_pages - move arbitrary anonymous pages of an existing vma * @ctx: pointer to the userfaultfd context diff --git a/mm/vma.c b/mm/vma.c index 6c35f0d775ab..f56317ee8247 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2644,6 +2644,23 @@ static int __mmap_new_file_vma(struct mmap_state *map, return 0; } +static void map_set_anon(struct mmap_state *map) +{ + map->file = NULL; + map->vm_ops = NULL; + map->pgoff = map->addr >> PAGE_SHIFT; +} + +static bool map_is_private(const struct mmap_state *map) +{ + return !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); +} + +static bool map_is_anon(const struct mmap_state *map) +{ + return map_is_private(map) && !map->file; +} + /* * __mmap_new_vma() - Allocate a new VMA for the region, as merging was not * possible. @@ -2657,8 +2674,7 @@ static int __mmap_new_file_vma(struct mmap_state *map, static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, struct mmap_action *action) { - const bool is_anon = !map->file && - !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); + const bool is_anon = map_is_anon(map); struct vma_iterator *vmi = map->vmi; int error = 0; struct vm_area_struct *vma; @@ -2800,6 +2816,10 @@ static int call_mmap_prepare(struct mmap_state *map, if (err) return err; + /* It's invalid for mmap_prepare hooks to clear vm_ops. */ + if (!desc->vm_ops) + return -EINVAL; + err = call_action_prepare(map, desc); if (err) return err; @@ -2816,16 +2836,21 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; + /* + * MAP_PRIVATE-/dev/zero mappings are an ancient way of getting + * anonymous mappings. Rather than allowing these mappings to be odd + * outliers, simply make them truly anonymous. + */ + if (map_is_private(map) && file_is_dev_zero(map->file)) + map_set_anon(map); + return 0; } static void set_vma_user_defined_fields(struct vm_area_struct *vma, struct mmap_state *map) { - if (map->vm_ops) - vma->vm_ops = map->vm_ops; - else /* Only /dev/zero should do this. */ - vma_set_anonymous(vma); + vma->vm_ops = map->vm_ops; vma->vm_private_data = map->vm_private_data; } @@ -2907,7 +2932,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, allocated_new = true; } - if (have_mmap_prepare && allocated_new) + if (have_mmap_prepare && !map_is_anon(&map) && allocated_new) set_vma_user_defined_fields(vma, &map); __mmap_complete(&map, vma); diff --git a/mm/vma.h b/mm/vma.h index 36973abaa015..f856d9ace3a6 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -267,9 +267,6 @@ static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) */ if (!vma_is_anonymous(vma)) return; - /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */ - if (vma->vm_file) - return; /* If faulted in, could have been remapped. */ if (vma->anon_vma) return; diff --git a/mm/vmalloc.c b/mm/vmalloc.c index bea9f76ed7e7..a009cc73f1b7 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3124,7 +3124,7 @@ EXPORT_SYMBOL(vm_map_ram); static struct vm_struct *vmlist __initdata; -static inline unsigned int vm_area_page_order(struct vm_struct *vm) +static inline unsigned int vm_area_page_order(const struct vm_struct *vm) { #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC return vm->page_order; @@ -3133,7 +3133,7 @@ static inline unsigned int vm_area_page_order(struct vm_struct *vm) #endif } -unsigned int get_vm_area_page_order(struct vm_struct *vm) +unsigned int get_vm_area_page_order(const struct vm_struct *vm) { return vm_area_page_order(vm); } @@ -3358,14 +3358,18 @@ struct vm_struct *remove_vm_area(const void *addr) } static inline void set_area_direct_map(const struct vm_struct *area, - int (*set_direct_map)(struct page *page)) + int (*set_direct_map)(struct page *page, + unsigned int nr)) { - unsigned long i; + unsigned int nr = (1U << vm_area_page_order(area)); - /* HUGE_VMALLOC passes small pages to set_direct_map */ - for (i = 0; i < area->nr_pages; i++) - if (page_address(area->pages[i])) - set_direct_map(area->pages[i]); + for (unsigned long i = 0; i < area->nr_pages; i += nr) { + if (page_address(area->pages[i])) { + int err = set_direct_map(area->pages[i], nr); + + WARN_ON_ONCE(err); + } + } } /* @@ -3872,7 +3876,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, unsigned long size = get_vm_area_size(area); unsigned long array_size; unsigned long nr_small_pages = size >> PAGE_SHIFT; - unsigned int page_order; + unsigned int page_order = page_shift - PAGE_SHIFT; unsigned int flags; int ret; @@ -3900,9 +3904,6 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, goto fail; } - set_vm_area_page_order(area, page_shift - PAGE_SHIFT); - page_order = vm_area_page_order(area); - /* * High-order nofail allocations are really expensive and * potentially dangerous (pre-mature OOM, disruptive reclaim @@ -3957,6 +3958,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, goto fail; } + set_vm_area_page_order(area, page_order); return area->addr; fail: @@ -4019,6 +4021,12 @@ static gfp_t vmalloc_fix_flags(gfp_t flags) * %__GFP_SKIP_KASAN can be used to skip unpoisoning of mapped pages * (when prot=%PAGE_KERNEL). * + * %VM_ALLOW_HUGE_VMAP allocates huge pages when possible and falls back to + * base pages if huge page allocation fails. + * + * %VM_REQUIRE_HUGE_VMAP implies %VM_ALLOW_HUGE_VMAP and fails instead of + * silently falling back to base pages. + * * Can not be called from interrupt nor NMI contexts. * Return: the address of the area or %NULL on failure */ @@ -4044,6 +4052,10 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, return NULL; } + /* VM_REQUIRE_HUGE_VMAP implies VM_ALLOW_HUGE_VMAP */ + if (vm_flags & VM_REQUIRE_HUGE_VMAP) + vm_flags |= VM_ALLOW_HUGE_VMAP; + if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { /* * Try huge pages. Only try for PAGE_KERNEL allocations, @@ -4060,6 +4072,9 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, align = max(original_align, 1UL << shift); } + if ((vm_flags & VM_REQUIRE_HUGE_VMAP) && shift == PAGE_SHIFT) + return NULL; + again: area = __get_vm_area_node(size, align, shift, VM_ALLOC | VM_UNINITIALIZED | vm_flags, start, end, node, @@ -4134,7 +4149,7 @@ again: return area->addr; fail: - if (shift > PAGE_SHIFT) { + if (shift > PAGE_SHIFT && !(vm_flags & VM_REQUIRE_HUGE_VMAP)) { shift = PAGE_SHIFT; align = original_align; goto again; @@ -5308,7 +5323,7 @@ static void show_purge_info(struct seq_file *m) for_each_vmap_node(vn) { spin_lock(&vn->lazy.lock); list_for_each_entry(va, &vn->lazy.head, list) { - seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", + seq_printf(m, "0x%p-0x%p %7ld unpurged vm_area\n", (void *)va->va_start, (void *)va->va_end, va_size(va)); } @@ -5316,7 +5331,7 @@ static void show_purge_info(struct seq_file *m) } } -static int vmalloc_info_show(struct seq_file *m, void *p) +static void show_busy_info(struct seq_file *m) { struct vmap_node *vn; struct vmap_area *va; @@ -5331,7 +5346,7 @@ static int vmalloc_info_show(struct seq_file *m, void *p) list_for_each_entry(va, &vn->busy.head, list) { if (!va->vm) { if (va->flags & VMAP_RAM) - seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", + seq_printf(m, "0x%p-0x%p %7ld vm_map_ram\n", (void *)va->va_start, (void *)va->va_end, va_size(va)); @@ -5345,7 +5360,7 @@ static int vmalloc_info_show(struct seq_file *m, void *p) /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ smp_rmb(); - seq_printf(m, "0x%pK-0x%pK %7ld", + seq_printf(m, "0x%p-0x%p %7ld", v->addr, v->addr + v->size, v->size); if (v->caller) @@ -5386,12 +5401,18 @@ static int vmalloc_info_show(struct seq_file *m, void *p) spin_unlock(&vn->busy.lock); } + if (IS_ENABLED(CONFIG_NUMA)) + kfree(counters); +} + +static int vmalloc_info_show(struct seq_file *m, void *p) +{ + show_busy_info(m); + /* * As a final step, dump "unpurged" areas. */ show_purge_info(m); - if (IS_ENABLED(CONFIG_NUMA)) - kfree(counters); return 0; } @@ -5404,11 +5425,23 @@ module_init(proc_vmalloc_init); #endif +static void __init vmap_insert_free_area(unsigned long start, unsigned long end) +{ + struct vmap_area *free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); + + if (!WARN_ON_ONCE(!free)) { + free->va_start = start; + free->va_end = end; + insert_vmap_area_augment(free, NULL, + &free_vmap_area_root, + &free_vmap_area_list); + } +} + static void __init vmap_init_free_space(void) { unsigned long vmap_start = 1; const unsigned long vmap_end = ULONG_MAX; - struct vmap_area *free; struct vm_struct *busy; /* @@ -5418,32 +5451,15 @@ static void __init vmap_init_free_space(void) * |<--------------------------------->| */ for (busy = vmlist; busy; busy = busy->next) { - if ((unsigned long) busy->addr - vmap_start > 0) { - free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); - if (!WARN_ON_ONCE(!free)) { - free->va_start = vmap_start; - free->va_end = (unsigned long) busy->addr; - - insert_vmap_area_augment(free, NULL, - &free_vmap_area_root, - &free_vmap_area_list); - } - } + if ((unsigned long) busy->addr - vmap_start > 0) + vmap_insert_free_area(vmap_start, + (unsigned long) busy->addr); vmap_start = (unsigned long) busy->addr + busy->size; } - if (vmap_end - vmap_start > 0) { - free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); - if (!WARN_ON_ONCE(!free)) { - free->va_start = vmap_start; - free->va_end = vmap_end; - - insert_vmap_area_augment(free, NULL, - &free_vmap_area_root, - &free_vmap_area_list); - } - } + if (vmap_end - vmap_start > 0) + vmap_insert_free_area(vmap_start, vmap_end); } static void vmap_init_nodes(void) @@ -5546,10 +5562,11 @@ void __init vmalloc_init(void) vbq = &per_cpu(vmap_block_queue, i); spin_lock_init(&vbq->lock); INIT_LIST_HEAD(&vbq->free); + xa_init(&vbq->vmap_blocks); + p = &per_cpu(vfree_deferred, i); init_llist_head(&p->list); INIT_WORK(&p->wq, delayed_vfree_work); - xa_init(&vbq->vmap_blocks); } /* diff --git a/mm/vmalloc.h b/mm/vmalloc.h index 8866ddcff668..211869f36509 100644 --- a/mm/vmalloc.h +++ b/mm/vmalloc.h @@ -12,7 +12,7 @@ void __init vmalloc_init(void); int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask); -unsigned int get_vm_area_page_order(struct vm_struct *vm); +unsigned int get_vm_area_page_order(const struct vm_struct *vm); #else static inline void vmalloc_init(void) {} diff --git a/mm/vmpressure.c b/mm/vmpressure.c index 9629240d77ad..3de99fef3928 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -30,9 +30,6 @@ * * As the vmscan reclaimer logic works with chunks which are multiple of * SWAP_CLUSTER_MAX, it makes sense to use it for the window size as well. - * - * TODO: Make the window size depend on machine size, as we do for vmstat - * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). */ const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; diff --git a/mm/vmscan.c b/mm/vmscan.c index f11491ee9ed5..37467004336b 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -182,17 +182,37 @@ struct scan_control { }; #ifdef ARCH_HAS_PREFETCHW -#define prefetchw_prev_lru_folio(_folio, _base, _field) \ - do { \ - if ((_folio)->lru.prev != _base) { \ - struct folio *prev; \ - \ - prev = lru_to_folio(&(_folio->lru)); \ - prefetchw(&prev->_field); \ - } \ - } while (0) +static inline void prefetchw_prev_lru_folio(struct folio *folio, + struct list_head *base) +{ + if (folio->lru.prev != base) { + struct folio *prev; + + prev = lru_to_folio(&folio->lru); + prefetchw(&prev->flags); + } +} + +static inline void prefetchw_next_lru_folio(struct folio *folio, + struct list_head *base) +{ + if (folio->lru.next != base) { + struct folio *next; + + next = list_entry(folio->lru.next, struct folio, lru); + prefetchw(&next->flags); + } +} #else -#define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) +static inline void prefetchw_prev_lru_folio(struct folio *folio, + struct list_head *base) +{ +} + +static inline void prefetchw_next_lru_folio(struct folio *folio, + struct list_head *base) +{ +} #endif /* @@ -342,20 +362,72 @@ static bool can_demote(int nid, struct scan_control *sc, return !nodes_empty(allowed_mask); } +#ifdef CONFIG_SWAP +static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg, + int nid, struct scan_control *sc) +{ + pg_data_t *pgdat = NODE_DATA(nid); + unsigned long anon_pages, swapcache; + + /* + * A GFP_NOIO reclaimer can only reclaim anon that is already in the + * swapcache (adding anon to the swapcache needs IO). When swapcache is + * far below the anon LRU, scanning anon reclaims nothing and only burns + * CPU. The 1/64 threshold keeps this to the case where anon is + * effectively unreclaimable. + */ + if (!sc || (sc->gfp_mask & __GFP_IO)) + return false; + + /* + * FIXME: MGLRU doesn't fully respect can_reclaim_anon_pages() for the + * scanning type, so only apply this to the traditional LRU for now. + */ + if (lru_gen_enabled()) + return false; + + if (memcg) { + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); + + anon_pages = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + + lruvec_page_state(lruvec, NR_ACTIVE_ANON); + swapcache = lruvec_page_state(lruvec, NR_SWAPCACHE); + } else { + anon_pages = node_page_state(pgdat, NR_INACTIVE_ANON) + + node_page_state(pgdat, NR_ACTIVE_ANON); + swapcache = node_page_state(pgdat, NR_SWAPCACHE); + } + + return swapcache < (anon_pages >> 6); +} +#else +static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg, + int nid, struct scan_control *sc) +{ + return true; +} +#endif /* CONFIG_SWAP */ + static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, int nid, struct scan_control *sc) { if (memcg == NULL) { /* - * For non-memcg reclaim, is there - * space in any swap device? + * For non-memcg reclaim, is there space in any swap device? + * And under GFP_NOIO, is there enough swapcached anon to make + * scanning anon worthwhile? */ - if (get_nr_swap_pages() > 0) + if (get_nr_swap_pages() > 0 && + !reclaimable_anon_is_low(memcg, nid, sc)) return true; } else { - /* Is the memcg below its swap limit? */ - if (mem_cgroup_get_nr_swap_pages(memcg) > 0) + /* + * Is the memcg below its swap limit, and under GFP_NOIO does + * it have enough swapcached anon to make scanning worthwhile? + */ + if (mem_cgroup_get_nr_swap_pages(memcg) > 0 && + !reclaimable_anon_is_low(memcg, nid, sc)) return true; } @@ -735,8 +807,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, if (reclaimed && !mapping_exiting(mapping)) shadow = workingset_eviction(folio, target_memcg); - __memcg1_swapout(folio, ci); - __swap_cache_del_folio(ci, folio, swap, shadow); + __swap_cache_del_folio(ci, folio, swap, shadow, true); swap_cluster_unlock_irq(ci); } else { void (*free_folio)(struct folio *); @@ -783,6 +854,22 @@ cannot_free: return 0; } +static long __remove_mapping_unfreeze(struct address_space *mapping, + struct folio *folio, bool reclaimed, + struct mem_cgroup *target_memcg) +{ + if (__remove_mapping(mapping, folio, reclaimed, target_memcg)) { + /* + * Unfreezing the refcount with 1 effectively + * drops the pagecache ref for us without requiring another + * atomic operation. + */ + folio_ref_unfreeze(folio, 1); + return folio_nr_pages(folio); + } + return 0; +} + /** * remove_mapping() - Attempt to remove a folio from its mapping. * @mapping: The address space. @@ -797,16 +884,29 @@ cannot_free: */ long remove_mapping(struct address_space *mapping, struct folio *folio) { - if (__remove_mapping(mapping, folio, false, NULL)) { - /* - * Unfreezing the refcount with 1 effectively - * drops the pagecache ref for us without requiring another - * atomic operation. - */ - folio_ref_unfreeze(folio, 1); - return folio_nr_pages(folio); - } - return 0; + return __remove_mapping_unfreeze(mapping, folio, false, NULL); +} + +/** + * remove_mapping_set_shadow() - Remove a folio and record an eviction shadow. + * @mapping: The address space. + * @folio: The folio to remove. + * @target_memcg: The memcg to charge the eviction shadow to; the caller must + * keep it alive across the call. + * + * Like remove_mapping(), but stores a workingset eviction shadow the way page + * reclaim does, so that a later refault can be detected and the folio + * re-activated. + * Return: The number of pages removed from the mapping. 0 if the folio + * could not be removed. + * Context: The caller should have a single refcount on the folio and + * hold its lock. + */ +long remove_mapping_set_shadow(struct address_space *mapping, + struct folio *folio, + struct mem_cgroup *target_memcg) +{ + return __remove_mapping_unfreeze(mapping, folio, true, target_memcg); } /** @@ -843,19 +943,22 @@ static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { /* Activate file-backed executable folios after first usage. */ if (is_exec_file_folio(folio, vma_flags)) { - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); + folio_set_workingset(folio); + folio_set_lru_refs(folio, 0); return true; } - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); + folio_set_lru_refs(folio, 1); return false; } /* Promote on second access */ - if (folio_lru_refs(folio) > 1) - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); - else + if (folio_lru_refs(folio) > 1) { + folio_set_workingset(folio); + folio_set_lru_refs(folio, 0); + } else { folio_mark_accessed(folio); + } return true; } #else @@ -955,7 +1058,7 @@ static void folio_check_dirty_writeback(struct folio *folio, *writeback = folio_test_writeback(folio); /* Verify dirty/writeback state if the filesystem supports it */ - if (!folio_test_private(folio)) + if (!folio_has_attached_private(folio)) return; mapping = folio_mapping(folio); @@ -1259,6 +1362,8 @@ retry: */ if (folio_test_anon(folio) && folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) { + int ret; + if (!(sc->gfp_mask & __GFP_IO)) goto keep_locked; if (folio_maybe_dma_pinned(folio)) @@ -1277,11 +1382,14 @@ retry: split_folio_to_list(folio, folio_list)) goto activate_locked; } - if (folio_alloc_swap(folio)) { + ret = folio_alloc_swap(folio); + if (ret) { int __maybe_unused order = folio_order(folio); if (!folio_test_large(folio)) goto activate_locked_split; + if (ret != -E2BIG) + goto activate_locked; /* Fallback to swap normal pages */ if (split_folio_to_list(folio, folio_list)) goto activate_locked; @@ -1695,7 +1803,7 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan, struct folio *folio; folio = lru_to_folio(src); - prefetchw_prev_lru_folio(folio, src, flags); + prefetchw_prev_lru_folio(folio, src); nr_pages = folio_nr_pages(folio); total_scan += nr_pages; @@ -1863,7 +1971,7 @@ static bool too_many_isolated(struct pglist_data *pgdat, int file, * * Note: The caller must not hold any lruvec lock. */ -static unsigned int move_folios_to_lru(struct list_head *list) +static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate) { int nr_pages, nr_moved = 0; struct lruvec *lruvec = NULL; @@ -1910,7 +2018,19 @@ static unsigned int move_folios_to_lru(struct list_head *list) continue; } - lruvec_add_folio(lruvec, folio); + /* + * Put clean, unreferenced and unpinned folios that may have + * missed folio_rotate_reclaimable() at the tail to avoid + * cold/hot inversion. + */ + if (do_rotate && !folio_test_active(folio) && !folio_mapped(folio) && + !folio_test_dirty(folio) && !folio_test_writeback(folio) && + !folio_test_referenced(folio) && + folio_ref_count(folio) == folio_expected_ref_count(folio)) + lruvec_add_folio_tail(lruvec, folio); + else + lruvec_add_folio(lruvec, folio); + nr_pages = folio_nr_pages(folio); nr_moved += nr_pages; if (folio_test_active(folio)) @@ -2027,7 +2147,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false, lruvec_memcg(lruvec)); - move_folios_to_lru(&folio_list); + move_folios_to_lru(&folio_list, true); mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), stat.nr_demoted); @@ -2138,8 +2258,8 @@ static void shrink_active_list(unsigned long nr_to_scan, /* * Move folios back to the lru list. */ - nr_activate = move_folios_to_lru(&l_active); - nr_deactivate = move_folios_to_lru(&l_inactive); + nr_activate = move_folios_to_lru(&l_active, false); + nr_deactivate = move_folios_to_lru(&l_inactive, false); count_vm_events(PGDEACTIVATE, nr_deactivate); count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); @@ -3195,8 +3315,11 @@ struct ctrl_pos { int gain; }; -static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, - struct ctrl_pos *pos) +/* + * __noipa works around gcc-16 warning for uninitialized use of pos->refaulted + */ +static void __noipa read_ctrl_pos(struct lruvec *lruvec, int type, int tier_min, + int tier_max, int gain, struct ctrl_pos *pos) { int i; struct lru_gen_folio *lrugen = &lruvec->lrugen; @@ -3205,7 +3328,7 @@ static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, pos->gain = gain; pos->refaulted = pos->total = 0; - for (i = tier % MAX_NR_TIERS; i <= min(tier, MAX_NR_TIERS - 1); i++) { + for (i = tier_min; i <= tier_max; i++) { pos->refaulted += lrugen->avg_refaulted[type][i] + atomic_long_read(&lrugen->refaulted[hist][type][i]); pos->total += lrugen->avg_total[type][i] + @@ -3266,11 +3389,11 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags) +static int folio_update_gen(struct folio *folio, int new_gen, int *type, + const vma_flags_t *vma_flags) { - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); - - VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + int old_gen; /* * See the comment on LRU_REFS_FLAGS, and activate file-backed @@ -3279,20 +3402,52 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma */ if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && !is_exec_file_folio(folio, vma_flags)) { - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); + folio_set_lru_refs(folio, 1); return -1; } do { + old_gen = lru_get_gen_flags(old_flags); + new_flags = old_flags; + /* lru_gen_del_folio() has isolated this page? */ - if (!(old_flags & LRU_GEN_MASK)) - return -1; + if (old_gen < 0) + break; - new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); - new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset); - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); + lru_set_gen_flags(&new_flags, new_gen); + lru_set_refs_flags(&new_flags, 0); + new_flags |= BIT(PG_workingset); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); - return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + *type = folio_flags_is_file_lru(&old_flags); + return old_gen; +} + +static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) +{ + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + int new_gen; + + do { + new_gen = lru_get_gen_flags(old_flags); + + /* folio_update_gen() has promoted this page? */ + if (new_gen >= 0 && new_gen != old_gen) { + if (increased) + *increased = false; + return new_gen; + } + + new_flags = old_flags; + new_gen = (old_gen + 1) % MAX_NR_GENS; + + lru_set_gen_flags(&new_flags, new_gen); + lru_set_refs_flags(&new_flags, 0); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); + + if (increased) + *increased = true; + return new_gen; } /* protect pages accessed multiple times through file descriptors */ @@ -3301,31 +3456,17 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) int type = folio_is_file_lru(folio); struct lru_gen_folio *lrugen = &lruvec->lrugen; int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); - - VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); - - do { - new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; - /* folio_update_gen() has promoted this page? */ - if (new_gen >= 0 && new_gen != old_gen) - return new_gen; - - new_gen = (old_gen + 1) % MAX_NR_GENS; - - new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); - new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); - - lru_gen_update_size(lruvec, folio, old_gen, new_gen); + bool gen_increased; + new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); + if (gen_increased) + lru_gen_update_size(lruvec, folio, old_gen, new_gen); return new_gen; } static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, - int old_gen, int new_gen) + int old_gen, int new_gen, int type) { - int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); @@ -3512,22 +3653,24 @@ static bool suitable_to_scan(int total, int young) } static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, - struct folio *folio, int new_gen, bool dirty) + struct lruvec *lruvec, struct folio *folio, bool dirty) { - int old_gen; + int new_gen, old_gen, type; if (!folio) return; + new_gen = lru_gen_from_seq(READ_ONCE(lruvec->lrugen.max_seq)); + if (dirty && !folio_test_dirty(folio) && !(folio_test_anon(folio) && folio_test_swapbacked(folio) && !folio_test_swapcache(folio))) folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen, &vma->flags); + old_gen = folio_update_gen(folio, new_gen, &type, &vma->flags); if (old_gen >= 0 && old_gen != new_gen) - update_batch_size(walk, folio, old_gen, new_gen); + update_batch_size(walk, folio, old_gen, new_gen, type); } else if (lru_gen_set_refs(folio, &vma->flags)) { old_gen = folio_lru_gen(folio); if (old_gen >= 0 && old_gen != new_gen) @@ -3549,8 +3692,6 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct lru_gen_mm_walk *walk = args->private; struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - DEFINE_MAX_SEQ(walk->lruvec); - int gen = lru_gen_from_seq(max_seq); unsigned int nr; pmd_t pmdval; @@ -3601,7 +3742,7 @@ restart: continue; if (last != folio) { - walk_update_folio(walk, args->vma, last, gen, dirty); + walk_update_folio(walk, args->vma, walk->lruvec, last, dirty); last = folio; dirty = false; @@ -3614,7 +3755,7 @@ restart: walk->mm_stats[MM_LEAF_YOUNG] += nr; } - walk_update_folio(walk, args->vma, last, gen, dirty); + walk_update_folio(walk, args->vma, walk->lruvec, last, dirty); last = NULL; if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) @@ -3637,8 +3778,6 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct lru_gen_mm_walk *walk = args->private; struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - DEFINE_MAX_SEQ(walk->lruvec); - int gen = lru_gen_from_seq(max_seq); VM_WARN_ON_ONCE(pud_leaf(*pud)); @@ -3692,7 +3831,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area goto next; if (last != folio) { - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, walk->lruvec, last, dirty); last = folio; dirty = false; @@ -3706,7 +3845,7 @@ next: i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, walk->lruvec, last, dirty); lazy_mmu_mode_disable(); spin_unlock(ptl); @@ -3897,6 +4036,19 @@ static void clear_mm_walk(void) kfree(walk); } +static inline void flush_lru_batch(struct list_head *head, struct list_head **batch_end, + struct list_head *dst) +{ + LIST_HEAD(movable); + + if (!*batch_end) + return; + + list_cut_position(&movable, head, *batch_end); + list_splice_tail_init(&movable, dst); + *batch_end = NULL; +} + static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) { int zone; @@ -3904,6 +4056,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) struct lru_gen_folio *lrugen = &lruvec->lrugen; int hist = lru_hist_from_seq(lrugen->min_seq[type]); int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + int target_gen = (old_gen + 1) % MAX_NR_GENS; /* For file type, skip the check if swappiness is anon only */ if (type && (swappiness == SWAPPINESS_ANON_ONLY)) @@ -3913,35 +4066,63 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) if (!type && !swappiness) goto done; + VM_WARN_ON_ONCE(get_nr_gens(lruvec, type) != MAX_NR_GENS); + VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) != + lru_gen_is_active(lruvec, target_gen)); /* prevent cold/hot inversion if the type is evictable */ for (zone = 0; zone < MAX_NR_ZONES; zone++) { + struct list_head *target_list = &lrugen->folios[target_gen][type][zone]; struct list_head *head = &lrugen->folios[old_gen][type][zone]; + struct list_head *pos = head->next; + struct list_head *batch_end = NULL; + long delta = 0; - while (!list_empty(head)) { - struct folio *folio = lru_to_folio(head); + while (pos != head) { + struct folio *folio = list_entry(pos, struct folio, lru); + long nr_pages = folio_nr_pages(folio); int refs = folio_lru_refs(folio); bool workingset = folio_test_workingset(folio); + bool gen_increased; VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); - new_gen = folio_inc_gen(lruvec, folio); - list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); + prefetchw_next_lru_folio(folio, head); + pos = pos->next; + new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); + /* + * If gen_increased is false, this is a promotion. Put folios + * at the head of the promoted gen. Otherwise, put them at + * the tail of the second-oldest gen. + */ + if (gen_increased) { + delta += nr_pages; + batch_end = &folio->lru; - /* don't count the workingset being lazily promoted */ - if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { - int tier = lru_tier_from_refs(refs, workingset); - int delta = folio_nr_pages(folio); + /* don't count the workingset being lazily promoted */ + if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { + int tier = lru_tier_from_refs(refs, workingset); - WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + delta); + WRITE_ONCE(lrugen->protected[hist][type][tier], + lrugen->protected[hist][type][tier] + nr_pages); + } + } else { + flush_lru_batch(head, &batch_end, target_list); + list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); } - if (!--remaining) - return false; + break; } + flush_lru_batch(head, &batch_end, target_list); + + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], + lrugen->nr_pages[old_gen][type][zone] - delta); + WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone], + lrugen->nr_pages[target_gen][type][zone] + delta); + if (!remaining) + return false; } done: reset_ctrl_pos(lruvec, type, true); @@ -4270,8 +4451,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) struct pglist_data *pgdat = folio_pgdat(folio); struct lruvec *lruvec; struct lru_gen_mm_state *mm_state; - unsigned long max_seq; - int gen; lockdep_assert_held(pvmw->ptl); VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); @@ -4308,8 +4487,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) memcg = get_mem_cgroup_from_folio(folio); lruvec = mem_cgroup_lruvec(memcg, pgdat); - max_seq = READ_ONCE((lruvec)->lrugen.max_seq); - gen = lru_gen_from_seq(max_seq); mm_state = get_mm_state(lruvec); lazy_mmu_mode_enable(); @@ -4341,7 +4518,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (last != folio) { - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, lruvec, last, dirty); last = folio; dirty = false; @@ -4353,13 +4530,14 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) young += nr; } - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, lruvec, last, dirty); lazy_mmu_mode_disable(); /* feedback from rmap walkers to page table walkers */ if (mm_state && suitable_to_scan(i, young)) - update_bloom_filter(mm_state, max_seq, pvmw->pmd); + update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq), + pvmw->pmd); mem_cgroup_put(memcg); @@ -4373,7 +4551,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* see the comment on MEMCG_NR_GENS */ enum { MEMCG_LRU_NOP, - MEMCG_LRU_HEAD, MEMCG_LRU_TAIL, MEMCG_LRU_OLD, MEMCG_LRU_YOUNG, @@ -4395,9 +4572,7 @@ static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) new = old = lruvec->lrugen.gen; /* see the comment on MEMCG_NR_GENS */ - if (op == MEMCG_LRU_HEAD) - seg = MEMCG_LRU_HEAD; - else if (op == MEMCG_LRU_TAIL) + if (op == MEMCG_LRU_TAIL) seg = MEMCG_LRU_TAIL; else if (op == MEMCG_LRU_OLD) new = get_memcg_gen(pgdat->memcg_lru.seq); @@ -4411,7 +4586,7 @@ static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) hlist_nulls_del_rcu(&lruvec->lrugen.list); - if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD) + if (op == MEMCG_LRU_OLD) hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); else hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); @@ -4489,15 +4664,6 @@ unlock: } } -void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) -{ - struct lruvec *lruvec = get_lruvec(memcg, nid); - - /* see the comment on MEMCG_NR_GENS */ - if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD) - lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD); -} - bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid) { struct lruvec *lruvec = get_lruvec(memcg, nid); @@ -4716,7 +4882,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio)) - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); + folio_set_lru_refs(folio, 0); success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); @@ -4802,16 +4968,16 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, static int get_tier_idx(struct lruvec *lruvec, int type) { int tier; - struct ctrl_pos sp, pv = {}; + struct ctrl_pos sp, pv; /* * To leave a margin for fluctuations, use a larger gain factor (2:3). * This value is chosen because any other tier would have at least twice * as many refaults as the first tier. */ - read_ctrl_pos(lruvec, type, 0, 2, &sp); - for (tier = 1; tier < MAX_NR_TIERS; tier++) { - read_ctrl_pos(lruvec, type, tier, 3, &pv); + read_ctrl_pos(lruvec, type, LRU_TIER_MIN, LRU_TIER_MIN, 2, &sp); + for (tier = LRU_TIER_MIN + 1; tier <= LRU_TIER_MAX; tier++) { + read_ctrl_pos(lruvec, type, tier, tier, 3, &pv); if (!positive_ctrl_err(&sp, &pv)) break; } @@ -4821,7 +4987,7 @@ static int get_tier_idx(struct lruvec *lruvec, int type) static int get_type_to_scan(struct lruvec *lruvec, int swappiness) { - struct ctrl_pos sp, pv = {}; + struct ctrl_pos sp, pv; if (swappiness <= MIN_SWAPPINESS + 1) return LRU_GEN_FILE; @@ -4832,41 +4998,59 @@ static int get_type_to_scan(struct lruvec *lruvec, int swappiness) * Compare the sum of all tiers of anon with that of file to determine * which type to scan. */ - read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp); - read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv); + read_ctrl_pos(lruvec, LRU_GEN_ANON, LRU_TIER_MIN, LRU_TIER_MAX, + swappiness, &sp); + read_ctrl_pos(lruvec, LRU_GEN_FILE, LRU_TIER_MIN, LRU_TIER_MAX, + MAX_SWAPPINESS - swappiness, &pv); return positive_ctrl_err(&sp, &pv); } +static inline bool is_single_type_reclaim(int swappiness) +{ + return swappiness == MIN_SWAPPINESS || + swappiness == SWAPPINESS_ANON_ONLY; +} + static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec, struct scan_control *sc, int swappiness, struct list_head *list, int *isolated, int *isolate_type, int *isolate_scanned) { - int i; - int total_scanned = 0; + bool type_fallback_allowed = !is_single_type_reclaim(swappiness); int type = get_type_to_scan(lruvec, swappiness); + int total_scanned = 0, scanned, tier; + bool tried = false; - for_each_evictable_type(i, swappiness) { - int scanned; - int tier = get_tier_idx(lruvec, type); +retry: + tier = get_tier_idx(lruvec, type); + scanned = scan_folios(nr_to_scan, lruvec, sc, + type, tier, list, isolated); - scanned = scan_folios(nr_to_scan, lruvec, sc, - type, tier, list, isolated); + total_scanned += scanned; + if (*isolated) { + *isolate_type = type; + *isolate_scanned = scanned; + return total_scanned; + } - total_scanned += scanned; - if (*isolated) { - *isolate_type = type; - *isolate_scanned = scanned; - break; - } - /* - * If scanned > 0 and isolated == 0, avoid falling back to the - * other type, as this type remains sufficient. Falling back - * too readily can disrupt the positive_ctrl_err() bias. - */ - if (!scanned) - type = !type; + /* + * We are running out of the current reclaim type. Fall back to + * the other type if allowed. + */ + if (!scanned && !tried && type_fallback_allowed) { + type = !type; + tried = true; + type_fallback_allowed = false; + goto retry; + } + /* + * We scanned some folios but failed to isolate any due to promotions, + * protections, or races. Retry once to avoid a larger loop. + */ + if (scanned && !tried) { + tried = true; + goto retry; } return total_scanned; @@ -4931,12 +5115,18 @@ retry: continue; } - /* don't add rejected folios to the oldest generation */ + /* + * See the comments on LRU_REFS_FLAGS. + * + * The rejected folios are never added to the oldest generation, + * so this effectively promotes them by at least one generation. + */ + folio_set_lru_refs(folio, 0); if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active)); + folio_set_active(folio); } - move_folios_to_lru(&list); + move_folios_to_lru(&list, false); walk = current->reclaim_state->mm_walk; if (walk && walk->batched) { @@ -5102,7 +5292,13 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); - /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ + /* + * For kswapd, mem_cgroup_calculate_protection() has already + * been called during the top-down cgroup traversal. + */ + if (!current_is_kswapd()) + mem_cgroup_calculate_protection_path(NULL, memcg); + if (mem_cgroup_below_min(NULL, memcg)) return MEMCG_LRU_YOUNG; @@ -5307,7 +5503,17 @@ static bool fill_evictable(struct lruvec *lruvec) while (!list_empty(head)) { bool success; - struct folio *folio = lru_to_folio(head); + struct folio *folio; + + /* + * lru_gen_add_folio() uses list_add_tail() rather + * than list_add() when reclaiming is true. Match + * its ordering to avoid cold/hot inversion. + */ + if (active) + folio = lru_to_folio(head); + else + folio = list_first_entry(head, struct folio, lru); VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); @@ -5315,7 +5521,11 @@ static bool fill_evictable(struct lruvec *lruvec) VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); lruvec_del_folio(lruvec, folio); - success = lru_gen_add_folio(lruvec, folio, false); + /* + * Borrow reclaiming=true to place inactive folios in + * the older gens. + */ + success = lru_gen_add_folio(lruvec, folio, !active); VM_WARN_ON_ONCE(!success); if (!--remaining) @@ -5670,6 +5880,9 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq, { DEFINE_MAX_SEQ(lruvec); + if (seq < max_seq) + return 0; + if (seq > max_seq) return -EINVAL; @@ -6425,8 +6638,6 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) { struct zoneref *z; struct zone *zone; - unsigned long nr_soft_reclaimed; - unsigned long nr_soft_scanned; gfp_t orig_mask; pg_data_t *last_pgdat = NULL; pg_data_t *first_pgdat = NULL; @@ -6468,35 +6679,17 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) sc->compaction_ready = true; continue; } - - /* - * Shrink each node in the zonelist once. If the - * zonelist is ordered by zone (not the default) then a - * node may be shrunk multiple times but in that case - * the user prefers lower zones being preserved. - */ - if (zone->zone_pgdat == last_pgdat) - continue; - - /* - * This steals pages from memory cgroups over softlimit - * and returns the number of reclaimed pages and - * scanned pages. This works for global memory pressure - * and balancing, not for a memcg's limit. - */ - nr_soft_scanned = 0; - nr_soft_reclaimed = memcg1_soft_limit_reclaim(zone->zone_pgdat, - sc->order, sc->gfp_mask, - &nr_soft_scanned); - sc->nr_reclaimed += nr_soft_reclaimed; - sc->nr_scanned += nr_soft_scanned; - /* need some check for avoid more shrink_zone() */ } if (!first_pgdat) first_pgdat = zone->zone_pgdat; - /* See comment about same check for global reclaim above */ + /* + * Shrink each node in the zonelist once. If the zonelist is + * ordered by zone (not the default) then a node may be shrunk + * multiple times but in that case the user prefers lower zones + * being preserved. + */ if (zone->zone_pgdat == last_pgdat) continue; last_pgdat = zone->zone_pgdat; @@ -6811,47 +7004,6 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, #ifdef CONFIG_MEMCG -/* Only used by soft limit reclaim. Do not reuse for anything else. */ -unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned) -{ - struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); - struct scan_control sc = { - .nr_to_reclaim = SWAP_CLUSTER_MAX, - .target_mem_cgroup = memcg, - .may_writepage = 1, - .may_unmap = 1, - .reclaim_idx = MAX_NR_ZONES - 1, - .may_swap = !noswap, - }; - - WARN_ON_ONCE(!current->reclaim_state); - - sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | - (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); - - trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.gfp_mask, - sc.order, - memcg); - - /* - * NOTE: Although we can get the priority field, using it - * here is not a good idea, since it limits the pages we can scan. - * if we don't reclaim here, the shrink_node from balance_pgdat - * will pick up pages from other mem cgroup's as well. We hack - * the priority and make it zero. - */ - shrink_lruvec(lruvec, &sc); - - trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed, memcg); - - *nr_scanned = sc.nr_scanned; - - return sc.nr_reclaimed; -} - unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, gfp_t gfp_mask, @@ -7157,8 +7309,6 @@ clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) { int i; - unsigned long nr_soft_reclaimed; - unsigned long nr_soft_scanned; unsigned long pflags; unsigned long nr_boost_reclaim; unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; @@ -7264,12 +7414,7 @@ restart: */ kswapd_age_node(pgdat, &sc); - /* Call soft limit reclaim before calling shrink_node. */ sc.nr_scanned = 0; - nr_soft_scanned = 0; - nr_soft_reclaimed = memcg1_soft_limit_reclaim(pgdat, sc.order, - sc.gfp_mask, &nr_soft_scanned); - sc.nr_reclaimed += nr_soft_reclaimed; /* * There should be no need to raise the scanning priority if @@ -7508,7 +7653,7 @@ static int kswapd(void *p) * us from recursively trying to free more memory as we're * trying to free the first piece of memory in the first place). */ - tsk->flags |= PF_MEMALLOC | PF_KSWAPD; + tsk->flags |= PF_MEMALLOC; set_freezable(); WRITE_ONCE(pgdat->kswapd_order, 0); @@ -7558,11 +7703,17 @@ kswapd_try_sleep: goto kswapd_try_sleep; } - tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); + tsk->flags &= ~PF_MEMALLOC; return 0; } +bool current_is_kswapd(void) +{ + return kthread_func(current) == kswapd; +} +EXPORT_SYMBOL_GPL(current_is_kswapd); + /* * A zone is low on free memory or too fragmented for high-order memory. If * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's @@ -7851,16 +8002,9 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, noreclaim_flag = memalloc_noreclaim_save(); set_task_reclaim_state(p, &sc->reclaim_state); - if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || - node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { - /* - * Free memory by calling shrink node with increasing - * priorities until we have enough memory freed. - */ - do { - shrink_node(pgdat, sc); - } while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0); - } + do { + shrink_node(pgdat, sc); + } while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0); set_task_reclaim_state(p, NULL); memalloc_noreclaim_restore(noreclaim_flag); diff --git a/mm/vmstat.c b/mm/vmstat.c index cb57714539fb..a3e809c57f29 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1837,6 +1837,11 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, struct per_cpu_zonestat __maybe_unused *pzstats; pcp = per_cpu_ptr(zone->per_cpu_pageset, i); + /* + * Access to the per-cpu pageset fields is lockless as they + * are used only for printing purposes. Use data_race to + * avoid KCSAN warning. + */ seq_printf(m, "\n cpu: %i" "\n count: %i" @@ -1845,15 +1850,15 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, "\n high_min: %i" "\n high_max: %i", i, - pcp->count, - pcp->high, - pcp->batch, - pcp->high_min, - pcp->high_max); + data_race(pcp->count), + data_race(pcp->high), + data_race(pcp->batch), + data_race(pcp->high_min), + data_race(pcp->high_max)); #ifdef CONFIG_SMP pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); seq_printf(m, "\n vm stats threshold: %d", - pzstats->stat_threshold); + data_race(pzstats->stat_threshold)); #endif } seq_printf(m, diff --git a/mm/workingset.c b/mm/workingset.c index 7ac2b88c80ae..8412f4840ae3 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -688,10 +688,9 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, mem_cgroup_flush_stats_ratelimited(sc->memcg); lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid)); - for (pages = 0, i = 0; i < NR_LRU_LISTS; i++) - pages += lruvec_lru_size(lruvec, i, MAX_NR_ZONES - 1); - + pages += lruvec_page_state_local(lruvec, + NR_LRU_BASE + i); pages += lruvec_page_state_local( lruvec, NR_SLAB_RECLAIMABLE_B) >> PAGE_SHIFT; pages += lruvec_page_state_local( diff --git a/mm/zpdesc.h b/mm/zpdesc.h index b8258dc78548..4fd81c2e8076 100644 --- a/mm/zpdesc.h +++ b/mm/zpdesc.h @@ -26,8 +26,8 @@ * with memcg_data. * * Page flags used: - * * PG_private identifies the first component page. * * PG_locked is used by page migration code. + * The first component page has zpdesc->zspage->first_zpdesc == zpdesc */ struct zpdesc { unsigned long flags; diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 825022a7a328..7ef80e0da626 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -290,11 +290,6 @@ struct zs_pool { atomic_t compaction_in_progress; }; -static inline void zpdesc_set_first(struct zpdesc *zpdesc) -{ - SetPagePrivate(zpdesc_page(zpdesc)); -} - static inline void zpdesc_inc_zone_page_state(struct zpdesc *zpdesc) { inc_zone_page_state(zpdesc_page(zpdesc), NR_ZSPAGES); @@ -476,11 +471,6 @@ static void record_obj(unsigned long handle, unsigned long obj) WRITE_ONCE(*(unsigned long *)handle, obj); } -static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc) -{ - return PagePrivate(zpdesc_page(zpdesc)); -} - /* Protected by class->lock */ static inline int get_zspage_inuse(struct zspage *zspage) { @@ -496,7 +486,8 @@ static struct zpdesc *get_first_zpdesc(struct zspage *zspage) { struct zpdesc *first_zpdesc = zspage->first_zpdesc; - VM_BUG_ON_PAGE(!is_first_zpdesc(first_zpdesc), zpdesc_page(first_zpdesc)); + /* the first zpdesc must point back to this zspage */ + VM_WARN_ON_ONCE_PAGE(first_zpdesc->zspage != zspage, zpdesc_page(first_zpdesc)); return first_zpdesc; } @@ -838,7 +829,8 @@ static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, struct zspage *zspage = get_zspage(zpdesc); if (unlikely(ZsHugePage(zspage))) { - VM_BUG_ON_PAGE(!is_first_zpdesc(zpdesc), zpdesc_page(zpdesc)); + /* only first zpdesc holds the handle */ + VM_WARN_ON_ONCE_PAGE(zspage->first_zpdesc != zpdesc, zpdesc_page(zpdesc)); handle = zpdesc->handle; } else handle = *(unsigned long *)obj; @@ -853,9 +845,6 @@ static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, static void reset_zpdesc(struct zpdesc *zpdesc) { - struct page *page = zpdesc_page(zpdesc); - - ClearPagePrivate(page); zpdesc->zspage = NULL; zpdesc->next = NULL; /* PageZsmalloc is sticky until the page is freed to the buddy. */ @@ -1006,8 +995,8 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage, * 1. all pages are linked together using zpdesc->next * 2. each sub-page point to zspage using zpdesc->zspage * - * we set PG_private to identify the first zpdesc (i.e. no other zpdesc - * has this flag set). + * The first zpdesc has its zspage->first_zpdesc set to itself, no + * other zpdesc has this set. */ for (i = 0; i < nr_zpdescs; i++) { zpdesc = zpdescs[i]; @@ -1015,7 +1004,6 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage, zpdesc->next = NULL; if (i == 0) { zspage->first_zpdesc = zpdesc; - zpdesc_set_first(zpdesc); if (unlikely(class->objs_per_zspage == 1 && class->pages_per_zspage == 1)) SetZsHugePage(zspage); @@ -1134,83 +1122,6 @@ unsigned long zs_get_total_pages(struct zs_pool *pool) } EXPORT_SYMBOL_GPL(zs_get_total_pages); -void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *local_copy) -{ - struct zspage *zspage; - struct zpdesc *zpdesc; - unsigned long obj, off; - unsigned int obj_idx; - struct size_class *class; - void *addr; - - /* Guarantee we can get zspage from handle safely */ - read_lock(&pool->lock); - obj = handle_to_obj(handle); - obj_to_location(obj, &zpdesc, &obj_idx); - zspage = get_zspage(zpdesc); - - /* Make sure migration doesn't move any pages in this zspage */ - zspage_read_lock(zspage); - read_unlock(&pool->lock); - - class = zspage_class(pool, zspage); - off = offset_in_page(class->size * obj_idx); - - if (!ZsHugePage(zspage)) - off += ZS_HANDLE_SIZE; - - if (off + mem_len <= PAGE_SIZE) { - /* this object is contained entirely within a page */ - addr = kmap_local_zpdesc(zpdesc); - addr += off; - } else { - size_t sizes[2]; - - /* this object spans two pages */ - sizes[0] = PAGE_SIZE - off; - sizes[1] = mem_len - sizes[0]; - addr = local_copy; - - memcpy_from_page(addr, zpdesc_page(zpdesc), - off, sizes[0]); - zpdesc = get_next_zpdesc(zpdesc); - memcpy_from_page(addr + sizes[0], - zpdesc_page(zpdesc), - 0, sizes[1]); - } - - return addr; -} -EXPORT_SYMBOL_GPL(zs_obj_read_begin); - -void zs_obj_read_end(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *handle_mem) -{ - struct zspage *zspage; - struct zpdesc *zpdesc; - unsigned long obj, off; - unsigned int obj_idx; - struct size_class *class; - - obj = handle_to_obj(handle); - obj_to_location(obj, &zpdesc, &obj_idx); - zspage = get_zspage(zpdesc); - class = zspage_class(pool, zspage); - off = offset_in_page(class->size * obj_idx); - - if (!ZsHugePage(zspage)) - off += ZS_HANDLE_SIZE; - - if (off + mem_len <= PAGE_SIZE) { - handle_mem -= off; - kunmap_local(handle_mem); - } - - zspage_read_unlock(zspage); -} -EXPORT_SYMBOL_GPL(zs_obj_read_end); - void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, struct scatterlist *sg, size_t mem_len) { diff --git a/mm/zswap.c b/mm/zswap.c index 37f34e406c8e..ae19e301fced 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -154,12 +155,23 @@ struct zswap_pool { struct zs_pool *zs_pool; struct crypto_acomp_ctx __percpu *acomp_ctx; struct percpu_ref ref; - struct list_head list; - struct work_struct release_work; + struct rcu_work release_rwork; struct hlist_node node; + u8 idx; char tfm_name[CRYPTO_MAX_ALG_NAME]; }; +/* + * Live pools keyed by id (1..ZSWAP_MAX_POOL_ID). XA_FLAGS_ALLOC1 keeps id 0 + * reserved so it is never handed to a live pool. XA_FLAGS_LOCK_BH makes the + * xa_lock softirq-safe: it is taken from __zswap_pool_empty(), which runs from + * a percpu_ref release callback in softirq context. + */ +#define ZSWAP_FIRST_POOL_ID 1 +#define ZSWAP_MAX_POOL_ID U8_MAX +static DEFINE_XARRAY_FLAGS(zswap_pools, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH); +static struct zswap_pool __rcu *zswap_current_pool; + /* Global LRU lists shared by all zswap pools. */ static struct list_lru zswap_list_lru; @@ -182,7 +194,7 @@ static struct shrinker *zswap_shrinker; * writeback logic. The entry is only reclaimed by the writeback * logic if referenced is unset. See comments in the shrinker * section for context. - * pool - the zswap_pool the entry's data is in + * pool_idx - id of the zswap_pool that the entry's data is in. * handle - zsmalloc allocation handle that stores the compressed page data * objcg - the obj_cgroup that the compressed memory is charged to * lru - handle to the pool's lru used to evict pages. @@ -191,19 +203,25 @@ struct zswap_entry { swp_entry_t swpentry; unsigned int length; bool referenced; - struct zswap_pool *pool; + u8 pool_idx; unsigned long handle; struct obj_cgroup *objcg; struct list_head lru; }; +/* + * No RCU section is needed around the returned pointer: a stored entry pins + * its pool via percpu_ref (taken in zswap_store_page()), so the id cannot be + * reused under us. Callers WARN and handle a NULL from a corrupt pool_idx. + */ +static struct zswap_pool *zswap_entry_pool(struct zswap_entry *entry) +{ + return xa_load(&zswap_pools, entry->pool_idx); +} + static struct xarray *zswap_trees[MAX_SWAPFILES]; static unsigned int nr_zswap_trees[MAX_SWAPFILES]; -/* RCU-protected iteration */ -static LIST_HEAD(zswap_pools); -/* protects zswap_pools list modification */ -static DEFINE_SPINLOCK(zswap_pools_lock); /* pool counter to provide unique names to zsmalloc */ static atomic_t zswap_pools_count = ATOMIC_INIT(0); @@ -228,10 +246,15 @@ static bool zswap_has_pool; /* One swap address space for each 64M swap space */ #define ZSWAP_ADDRESS_SPACE_SHIFT 14 #define ZSWAP_ADDRESS_SPACE_PAGES (1 << ZSWAP_ADDRESS_SPACE_SHIFT) + +static inline struct xarray *zswap_tree(int type, pgoff_t offset) +{ + return &zswap_trees[type][offset >> ZSWAP_ADDRESS_SPACE_SHIFT]; +} + static inline struct xarray *swap_zswap_tree(swp_entry_t swp) { - return &zswap_trees[swp_type(swp)][swp_offset(swp) - >> ZSWAP_ADDRESS_SPACE_SHIFT]; + return zswap_tree(swp_type(swp), swp_offset(swp)); } #define zswap_pool_debug(msg, p) \ @@ -275,6 +298,7 @@ static struct zswap_pool *zswap_pool_create(char *compressor) struct zswap_pool *pool; char name[38]; /* 'zswap' + 32 char (max) num + \0 */ int ret, cpu; + u32 id; if (!zswap_has_pool && !strcmp(compressor, ZSWAP_PARAM_UNSET)) return NULL; @@ -320,12 +344,31 @@ static struct zswap_pool *zswap_pool_create(char *compressor) PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); if (ret) goto ref_fail; - INIT_LIST_HEAD(&pool->list); + + /* + * Publish only after the pool is fully built, so lockless walkers + * never see a half-initialized pool. The _bh variant pairs with the + * softirq-context xa_lock taken in __zswap_pool_empty(). + */ + ret = xa_alloc_bh(&zswap_pools, &id, pool, + XA_LIMIT(ZSWAP_FIRST_POOL_ID, ZSWAP_MAX_POOL_ID), + GFP_KERNEL); + if (ret) { + if (ret == -EBUSY) + pr_err("cannot allocate pool id (max %d live pools)\n", + ZSWAP_MAX_POOL_ID - ZSWAP_FIRST_POOL_ID + 1); + goto xa_fail; + } + pool->idx = id; zswap_pool_debug("created", pool); + static_branch_enable(&zswap_ever_enabled); + return pool; +xa_fail: + percpu_ref_exit(&pool->ref); ref_fail: cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); @@ -379,37 +422,29 @@ static void zswap_pool_destroy(struct zswap_pool *pool) static void __zswap_pool_release(struct work_struct *work) { - struct zswap_pool *pool = container_of(work, typeof(*pool), - release_work); - - synchronize_rcu(); + struct zswap_pool *pool = container_of(to_rcu_work(work), + typeof(*pool), release_rwork); /* nobody should have been able to get a ref... */ WARN_ON(!percpu_ref_is_zero(&pool->ref)); percpu_ref_exit(&pool->ref); - /* pool is now off zswap_pools list and has no references. */ + /* The pool is no longer in zswap_pools and has no references. */ zswap_pool_destroy(pool); } -static struct zswap_pool *zswap_pool_current(void); - static void __zswap_pool_empty(struct percpu_ref *ref) { struct zswap_pool *pool; pool = container_of(ref, typeof(*pool), ref); - spin_lock_bh(&zswap_pools_lock); + WARN_ON(pool == rcu_access_pointer(zswap_current_pool)); - WARN_ON(pool == zswap_pool_current()); + xa_erase_bh(&zswap_pools, pool->idx); - list_del_rcu(&pool->list); - - INIT_WORK(&pool->release_work, __zswap_pool_release); - schedule_work(&pool->release_work); - - spin_unlock_bh(&zswap_pools_lock); + INIT_RCU_WORK(&pool->release_rwork, __zswap_pool_release); + queue_rcu_work(system_percpu_wq, &pool->release_rwork); } static int __must_check zswap_pool_tryget(struct zswap_pool *pool) @@ -435,20 +470,13 @@ static struct zswap_pool *__zswap_pool_current(void) { struct zswap_pool *pool; - pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); + pool = rcu_dereference(zswap_current_pool); WARN_ONCE(!pool && zswap_has_pool, "%s: no page storage pool!\n", __func__); return pool; } -static struct zswap_pool *zswap_pool_current(void) -{ - assert_spin_locked(&zswap_pools_lock); - - return __zswap_pool_current(); -} - static struct zswap_pool *zswap_pool_current_get(void) { struct zswap_pool *pool; @@ -464,23 +492,28 @@ static struct zswap_pool *zswap_pool_current_get(void) return pool; } -/* type and compressor must be null-terminated */ +/* compressor must be null-terminated */ static struct zswap_pool *zswap_pool_find_get(char *compressor) { struct zswap_pool *pool; + unsigned long id; - assert_spin_locked(&zswap_pools_lock); - - list_for_each_entry_rcu(pool, &zswap_pools, list) { + /* + * __zswap_pool_empty() can erase from zswap_pools in softirq while we + * walk. rcu_read_lock() keeps the walk consistent and each pool alive + * across tryget(). xa_for_each()'s own RCU does not span the loop body. + */ + rcu_read_lock(); + xa_for_each(&zswap_pools, id, pool) { if (strcmp(pool->tfm_name, compressor)) continue; /* if we can't get it, it's about to be destroyed */ - if (!zswap_pool_tryget(pool)) - continue; - return pool; + if (zswap_pool_tryget(pool)) + break; } + rcu_read_unlock(); - return NULL; + return pool; } static unsigned long zswap_max_pages(void) @@ -497,9 +530,14 @@ unsigned long zswap_total_pages(void) { struct zswap_pool *pool; unsigned long total = 0; + unsigned long id; + /* + * rcu_read_lock() keeps each pool alive across zs_get_total_pages(). + * xa_for_each()'s own RCU does not span the loop body. + */ rcu_read_lock(); - list_for_each_entry_rcu(pool, &zswap_pools, list) + xa_for_each(&zswap_pools, id, pool) total += zs_get_total_pages(pool->zs_pool); rcu_read_unlock(); @@ -556,20 +594,13 @@ static int zswap_compressor_param_set(const char *val, const struct kernel_param return -ENOENT; } - spin_lock_bh(&zswap_pools_lock); - pool = zswap_pool_find_get(s); - if (pool) { - zswap_pool_debug("using existing", pool); - WARN_ON(pool == zswap_pool_current()); - list_del_rcu(&pool->list); - } - - spin_unlock_bh(&zswap_pools_lock); - - if (!pool) + if (!pool) { pool = zswap_pool_create(s); - else { + } else { + zswap_pool_debug("using existing", pool); + WARN_ON(pool == rcu_access_pointer(zswap_current_pool)); + /* * Restore the initial ref dropped by percpu_ref_kill() * when the pool was decommissioned and switch it again @@ -586,24 +617,18 @@ static int zswap_compressor_param_set(const char *val, const struct kernel_param else ret = -EINVAL; - spin_lock_bh(&zswap_pools_lock); - + /* + * Compressor switches are serialized by the kernel param lock, so this + * is the only writer of zswap_current_pool: no xa_lock needed. + */ if (!ret) { - put_pool = zswap_pool_current(); - list_add_rcu(&pool->list, &zswap_pools); + put_pool = rcu_access_pointer(zswap_current_pool); + rcu_assign_pointer(zswap_current_pool, pool); zswap_has_pool = true; } else if (pool) { - /* - * Add the possibly pre-existing pool to the end of the pools - * list; if it's new (and empty) then it'll be removed and - * destroyed by the put after we drop the lock - */ - list_add_tail_rcu(&pool->list, &zswap_pools); put_pool = pool; } - spin_unlock_bh(&zswap_pools_lock); - /* * Drop the ref from either the old current pool, * or the new pool we failed to add @@ -647,19 +672,6 @@ static int zswap_enabled_param_set(const char *val, * lru functions **********************************/ -/* should be called under RCU */ -#ifdef CONFIG_MEMCG -static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) -{ - return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL; -} -#else -static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) -{ - return NULL; -} -#endif - static inline int entry_to_nid(struct zswap_entry *entry) { return page_to_nid(virt_to_page(entry)); @@ -682,7 +694,7 @@ static void zswap_lru_add(struct zswap_entry *entry) * Similar reasoning holds for list_lru_del(). */ rcu_read_lock(); - memcg = mem_cgroup_from_entry(entry); + memcg = obj_cgroup_memcg(entry->objcg); /* will always succeed */ list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg); rcu_read_unlock(); @@ -694,7 +706,7 @@ static void zswap_lru_del(struct zswap_entry *entry) struct mem_cgroup *memcg; rcu_read_lock(); - memcg = mem_cgroup_from_entry(entry); + memcg = obj_cgroup_memcg(entry->objcg); /* will always succeed */ list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg); rcu_read_unlock(); @@ -764,9 +776,13 @@ static void zswap_entry_cache_free(struct zswap_entry *entry) */ static void zswap_entry_free(struct zswap_entry *entry) { + struct zswap_pool *pool = zswap_entry_pool(entry); + zswap_lru_del(entry); - zs_free(entry->pool->zs_pool, entry->handle); - zswap_pool_put(entry->pool); + if (!WARN_ON_ONCE(!pool)) { + zs_free(pool->zs_pool, entry->handle); + zswap_pool_put(pool); + } if (entry->objcg) { obj_cgroup_uncharge_zswap(entry->objcg, entry->length); obj_cgroup_put(entry->objcg); @@ -837,8 +853,8 @@ fail: return ret; } -static bool zswap_compress(struct page *page, struct zswap_entry *entry, - struct zswap_pool *pool) +static bool zswap_compress(struct folio *folio, long index, + struct zswap_entry *entry, struct zswap_pool *pool) { struct crypto_acomp_ctx *acomp_ctx; struct scatterlist input, output; @@ -854,7 +870,7 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, dst = acomp_ctx->buffer; sg_init_table(&input, 1); - sg_set_page(&input, page, PAGE_SIZE, 0); + sg_set_folio(&input, folio, PAGE_SIZE, index * PAGE_SIZE); sg_init_one(&output, dst, PAGE_SIZE); acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen); @@ -883,8 +899,7 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, */ if (comp_ret || !dlen || dlen >= PAGE_SIZE) { rcu_read_lock(); - if (!mem_cgroup_zswap_writeback_enabled( - folio_memcg(page_folio(page)))) { + if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio))) { rcu_read_unlock(); comp_ret = comp_ret ? comp_ret : -EINVAL; goto unlock; @@ -892,12 +907,12 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, rcu_read_unlock(); comp_ret = 0; dlen = PAGE_SIZE; - dst = kmap_local_page(page); + dst = kmap_local_folio(folio, index * PAGE_SIZE); mapped = true; } gfp = GFP_NOWAIT | __GFP_NORETRY | __GFP_HIGHMEM | __GFP_MOVABLE; - handle = zs_malloc(pool->zs_pool, dlen, gfp, page_to_nid(page)); + handle = zs_malloc(pool->zs_pool, dlen, gfp, folio_nid(folio)); if (IS_ERR_VALUE(handle)) { alloc_ret = PTR_ERR((void *)handle); goto unlock; @@ -923,12 +938,15 @@ unlock: static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio) { - struct zswap_pool *pool = entry->pool; + struct zswap_pool *pool = zswap_entry_pool(entry); struct scatterlist input[2]; /* zsmalloc returns an SG list 1-2 entries */ struct scatterlist output; struct crypto_acomp_ctx *acomp_ctx; int ret = 0, dlen; + if (WARN_ON_ONCE(!pool)) + return false; + acomp_ctx = raw_cpu_ptr(pool->acomp_ctx); mutex_lock(&acomp_ctx->mutex); zs_obj_read_sg_begin(pool->zs_pool, entry->handle, input, entry->length); @@ -964,7 +982,7 @@ static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio) pr_alert_ratelimited("Decompression error from zswap (%d:%lu %s %u->%d)\n", swp_type(entry->swpentry), swp_offset(entry->swpentry), - entry->pool->tfm_name, + pool->tfm_name, entry->length, dlen); return false; } @@ -997,12 +1015,12 @@ static int zswap_writeback_entry(struct zswap_entry *entry, /* try to allocate swap cache folio */ si = get_swap_device(swpentry); - if (!si) - return -EEXIST; + if (IS_ERR_OR_NULL(si)) + return -ENOENT; mpol = get_task_policy(current); - folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, - NO_INTERLEAVE_INDEX); + folio = __swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, + NO_INTERLEAVE_INDEX); put_swap_device(si); /* @@ -1015,6 +1033,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry, if (IS_ERR(folio)) return PTR_ERR(folio); + folio_clear_active(folio); + /* * folio is locked, and the swapcache is now secured against * concurrent swapping to and from the slot, and concurrent @@ -1027,12 +1047,12 @@ static int zswap_writeback_entry(struct zswap_entry *entry, tree = swap_zswap_tree(swpentry); if (entry != xa_load(tree, offset)) { ret = -ENOMEM; - goto out; + goto err; } if (!zswap_decompress(entry, folio)) { ret = -EIO; - goto out; + goto err; } xa_erase(tree, offset); @@ -1046,18 +1066,30 @@ static int zswap_writeback_entry(struct zswap_entry *entry, /* folio is up to date */ folio_mark_uptodate(folio); - /* move it to the tail of the inactive list after end_writeback */ - folio_set_reclaim(folio); + folio_set_dropbehind(folio); + + /* + * Drop our reference before starting writeback so the swap cache holds + * the only one: the drop in folio_end_writeback() needs that for + * remove_mapping_set_shadow() to succeed, otherwise the folio is + * handed back to reclaim instead. + * + * Nothing can free the folio in the meantime: we hold the folio lock + * until writeback starts, PG_writeback then blocks swap cache removal, + * and folio_end_writeback() takes its own reference before clearing + * PG_writeback and donates it to the drop. + */ + folio_put(folio); /* start writeback */ - __swap_writepage(&ctx, folio); + __swap_writeout(&ctx, folio); swap_write_submit(&ctx); -out: - if (ret) { - swap_cache_del_folio(folio); - folio_unlock(folio); - } + return 0; + +err: + swap_cache_del_folio(folio); + folio_unlock(folio); folio_put(folio); return ret; } @@ -1405,23 +1437,30 @@ resched: * main API **********************************/ -static bool zswap_store_page(struct page *page, +static bool zswap_store_page(struct folio *folio, long index, struct obj_cgroup *objcg, struct zswap_pool *pool) { - swp_entry_t page_swpentry = page_swap_entry(page); + swp_entry_t page_swpentry = folio_swap_entry(folio, index); struct zswap_entry *entry, *old; /* allocate entry */ - entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page)); + entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio)); if (!entry) { zswap_reject_kmemcache_fail++; return false; } - if (!zswap_compress(page, entry, pool)) + if (!zswap_compress(folio, index, entry, pool)) goto compress_failed; + /* + * Set pool_idx before the xa_store() below publishes the entry, or a + * concurrent reader could resolve a stale pool_idx left by slab reuse + * to an unrelated live pool. + */ + entry->pool_idx = pool->idx; + old = xa_store(swap_zswap_tree(page_swpentry), swp_offset(page_swpentry), entry, GFP_KERNEL); @@ -1467,7 +1506,6 @@ static bool zswap_store_page(struct page *page, * The publishing order matters to prevent writeback from seeing * an incoherent entry. */ - entry->pool = pool; entry->swpentry = page_swpentry; entry->objcg = objcg; entry->referenced = true; @@ -1528,9 +1566,7 @@ bool zswap_store(struct folio *folio) } for (index = 0; index < nr_pages; ++index) { - struct page *page = folio_page(folio, index); - - if (!zswap_store_page(page, objcg, pool)) + if (!zswap_store_page(folio, index, objcg, pool)) goto put_pool; } @@ -1554,23 +1590,38 @@ check_old: * offsets corresponding to each page of the folio. Otherwise, * writeback could overwrite the new data in the swapfile. */ - if (!ret) { - unsigned type = swp_type(swp); - pgoff_t offset = swp_offset(swp); - struct zswap_entry *entry; - struct xarray *tree; - - for (index = 0; index < nr_pages; ++index) { - tree = swap_zswap_tree(swp_entry(type, offset + index)); - entry = xa_erase(tree, offset + index); - if (entry) - zswap_entry_free(entry); - } - } + if (!ret) + zswap_invalidate(swp_type(swp), swp_offset(swp), nr_pages); return ret; } +/** + * zswap_is_present() - is any slot in [entry, entry + nr) in zswap? + * @entry: base swap entry of the range + * @nr: number of contiguous slots to check + * + * Context: The caller must keep the range pinned, otherwise the answer can + * change under it. + * Return: true if at least one slot in the range is in zswap. + */ +static bool zswap_is_present(swp_entry_t entry, unsigned int nr) +{ + pgoff_t offset = swp_offset(entry); + struct xarray *tree = swap_zswap_tree(entry); + unsigned long index = offset; + + /* + * A pinned range is at most SWAPFILE_CLUSTER slots and is aligned to + * its own size, so one tree covers all of it and a single lookup is + * enough. Scanning only part of the range would report a false + * "absent" and let the caller read a stale copy from the device. + */ + BUILD_BUG_ON(SWAPFILE_CLUSTER > ZSWAP_ADDRESS_SPACE_PAGES); + + return xa_find(tree, &index, offset + nr - 1, XA_PRESENT); +} + /** * zswap_load() - load a folio from zswap * @folio: folio to load @@ -1578,15 +1629,12 @@ check_old: * Return: 0 on success, with the folio unlocked and marked up-to-date, or one * of the following error codes: * - * -EIO: if the swapped out content was in zswap, but could not be loaded - * into the page due to a decompression failure. The folio is unlocked, but - * NOT marked up-to-date, so that an IO error is emitted (e.g. do_swap_page() - * will SIGBUS). - * - * -EINVAL: if the swapped out content was in zswap, but the page belongs - * to a large folio, which is not supported by zswap. The folio is unlocked, - * but NOT marked up-to-date, so that an IO error is emitted (e.g. - * do_swap_page() will SIGBUS). + * -EIO: if the swapped out content was in zswap but could not be handed + * back, either because decompression failed or because a slot in a + * large-folio range is still in zswap and zswap cannot reconstruct a large + * folio from per-page entries. The folio is unlocked, but NOT marked + * up-to-date, so that an IO error is emitted (e.g. do_swap_page() will + * SIGBUS). * * -ENOENT: if the swapped out content was not in zswap. The folio remains * locked on return. @@ -1605,13 +1653,18 @@ int zswap_load(struct folio *folio) return -ENOENT; /* - * Large folios should not be swapped in while zswap is being used, as - * they are not properly handled. Zswap does not properly load large - * folios, and a large folio may only be partially in zswap. + * A large folio can legitimately reach zswap_load() with its whole + * range on the backing device, so scan the range rather than rejecting + * it outright. The caller has pinned every slot, so zswap cannot start + * a store or a writeback into the range while we look. */ - if (WARN_ON_ONCE(folio_test_large(folio))) { - folio_unlock(folio); - return -EINVAL; + if (folio_test_large(folio)) { + if (WARN_ON_ONCE(zswap_is_present(swp, + folio_nr_pages(folio)))) { + folio_unlock(folio); + return -EIO; + } + return -ENOENT; } entry = xa_load(tree, offset); @@ -1644,18 +1697,25 @@ int zswap_load(struct folio *folio) return 0; } -void zswap_invalidate(swp_entry_t swp) +void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries) { - pgoff_t offset = swp_offset(swp); - struct xarray *tree = swap_zswap_tree(swp); struct zswap_entry *entry; + struct xarray *tree; + unsigned long i; - if (xa_empty(tree)) + if (zswap_never_enabled()) return; - entry = xa_erase(tree, offset); - if (entry) - zswap_entry_free(entry); + for (i = 0; i < nr_entries; i++) { + tree = zswap_tree(type, offset + i); + + if (xa_empty(tree)) + continue; + + entry = xa_erase(tree, offset + i); + if (entry) + zswap_entry_free(entry); + } } int zswap_swapon(int type, unsigned long nr_pages) @@ -1803,9 +1863,8 @@ static int zswap_setup(void) pool = __zswap_pool_create_fallback(); if (pool) { pr_info("loaded using pool %s\n", pool->tfm_name); - list_add(&pool->list, &zswap_pools); + rcu_assign_pointer(zswap_current_pool, pool); zswap_has_pool = true; - static_branch_enable(&zswap_ever_enabled); } else { pr_err("pool creation failed\n"); zswap_enabled = false; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 99d7237a623d..3fd477605704 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2172,27 +2172,18 @@ static void tcp_zc_finalize_rx_tstamp(struct sock *sk, } static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm, - unsigned long address, - bool *mmap_locked) + unsigned long address) { - struct vm_area_struct *vma = lock_vma_under_rcu(mm, address); + struct vm_area_struct *vma = vma_start_read_unlocked(mm, address); - if (vma) { - if (vma->vm_ops != &tcp_vm_ops) { - vma_end_read(vma); - return NULL; - } - *mmap_locked = false; - return vma; - } + if (!vma) + return NULL; - mmap_read_lock(mm); - vma = vma_lookup(mm, address); - if (!vma || vma->vm_ops != &tcp_vm_ops) { - mmap_read_unlock(mm); + if (vma->vm_ops != &tcp_vm_ops) { + vma_end_read(vma); return NULL; } - *mmap_locked = true; + return vma; } @@ -2213,7 +2204,6 @@ static int tcp_zerocopy_receive(struct sock *sk, u32 seq = tp->copied_seq; u32 total_bytes_to_map; int inq = tcp_inq(sk); - bool mmap_locked; int ret; zc->copybuf_len = 0; @@ -2238,7 +2228,7 @@ static int tcp_zerocopy_receive(struct sock *sk, return 0; } - vma = find_tcp_vma(current->mm, address, &mmap_locked); + vma = find_tcp_vma(current->mm, address); if (!vma) return -EINVAL; @@ -2320,10 +2310,7 @@ static int tcp_zerocopy_receive(struct sock *sk, zc, total_bytes_to_map); } out: - if (mmap_locked) - mmap_read_unlock(current->mm); - else - vma_end_read(vma); + vma_end_read(vma); /* Try to copy straggler data. */ if (!ret) copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 4764d7b68f2a..58bc1793fdaf 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -170,30 +170,48 @@ impl MmWithUser { /// /// This is an optimistic trylock operation, so it may fail if there is contention. In that /// case, you should fall back to taking the mmap read lock. - /// - /// When per-vma locks are disabled, this always returns `None`. #[inline] pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option> { - #[cfg(CONFIG_PER_VMA_LOCK)] - { - // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where - // `mm_users` is non-zero. - let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; - if !vma.is_null() { - return Some(VmaReadGuard { - // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a - // valid vma. The vma is stable for as long as the vma read lock is held. - vma: unsafe { VmaRef::from_raw(vma) }, - _nts: NotThreadSafe, - }); - } + // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where + // `mm_users` is non-zero. + let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; + if vma.is_null() { + return None; } + Some(VmaReadGuard { + // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }) + } - // Silence warnings about unused variables. - #[cfg(not(CONFIG_PER_VMA_LOCK))] - let _ = vma_addr; - - None + /// Find the VMA covering 'address' and read-lock it. + /// + /// The fast path does not take mmap_lock. Waits for writers to finish if the + /// VMA is being modified by taking mmap_lock. + /// Use when mmap_lock is not held, otherwise use vma_start_read_locked(). + /// Nothing prevents VMAs being unmapped/mapped before or after the VMA is + /// looked up, if a stronger guarantee is required, take an mmap_lock. + /// + /// Return: If a VMA exists which spans @address, return that VMA, read-locked. + /// If no VMA is mapped there or, very unlikely, a reference count overflow + /// occurred, return NULL. + #[inline] + pub fn vma_start_read_unlocked(&self, vma_addr: usize) -> Option> { + // SAFETY: We may invoke `vma_start_read_unlocked` because we know this `mm` has non-zero + // `mm_users`. + let vma = unsafe { bindings::vma_start_read_unlocked(self.as_raw(), vma_addr) }; + if vma.is_null() { + return None; + } + // INVARIANT: We just acquired the VMA read lock. + Some(VmaReadGuard { + // SAFETY: If `vma_start_read_unlocked` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }) } /// Lock the mmap read lock. diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index d1123ebbfab9..27dc88bdf7a0 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -52,6 +52,11 @@ module_param(detect_node_addresses, bool, 0600); static struct damon_ctx *ctxs[2]; +/* + * Use phys_addr_t instead of damon_addr_range (unsigned long) for physical + * addresses. On 32-bit systems with more than 4GB memory, phys_addr_t will + * be 64-bit while unsigned long is 32-bit. + */ struct region_range { phys_addr_t start; phys_addr_t end; @@ -156,9 +161,6 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) if (!scheme) goto free_out; damon_set_schemes(ctx, &scheme, 1); - /* zero target value causes division by zero in damos_quota_store() */ - if (!node0_mem_used_bp || !node0_mem_free_bp) - goto free_out; quota_goal = damos_new_quota_goal( promote ? DAMOS_QUOTA_NODE_MEM_USED_BP : DAMOS_QUOTA_NODE_MEM_FREE_BP, diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index 842099bd6228..83ddf12811d5 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -32,7 +32,6 @@ module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_PRCL"); static struct damon_ctx *ctx; -static struct pid *target_pidp; static int damon_sample_prcl_repeat_call_fn(void *data) { @@ -79,12 +78,10 @@ static int damon_sample_prcl_start(void) return -ENOMEM; } damon_add_target(ctx, target); - target_pidp = find_get_pid(target_pid); - if (!target_pidp) { + if (damon_set_target_pid(target, target_pid)) { damon_destroy_ctx(ctx); return -EINVAL; } - target->pid = target_pidp; scheme = damon_new_scheme( &(struct damos_access_pattern) { diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index 37fd5da20158..53944aea8428 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -33,7 +33,6 @@ module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_WSSE"); static struct damon_ctx *ctx; -static struct pid *target_pidp; static int damon_sample_wsse_repeat_call_fn(void *data) { @@ -79,12 +78,10 @@ static int damon_sample_wsse_start(void) return -ENOMEM; } damon_add_target(ctx, target); - target_pidp = find_get_pid(target_pid); - if (!target_pidp) { + if (damon_set_target_pid(target, target_pid)) { damon_destroy_ctx(ctx); return -EINVAL; } - target->pid = target_pidp; err = damon_start(&ctx, 1, true); if (err) { diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py index dffadccbb01d..193a88d763ab 100644 --- a/scripts/gdb/linux/mm.py +++ b/scripts/gdb/linux/mm.py @@ -56,7 +56,7 @@ class x86_page_ops(): self.MAX_PHYSMEM_BITS = 46 self.SECTION_SIZE_BITS = 27 - self.MAX_ORDER = 10 + self.MAX_PAGE_ORDER = 10 self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT @@ -70,7 +70,6 @@ class x86_page_ops(): self.SECTIONS_PER_ROOT = 1 self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT) - self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1 try: self.SECTION_HAS_MEM_MAP = 1 << int(gdb.parse_and_eval('SECTION_HAS_MEM_MAP_BIT')) @@ -100,7 +99,7 @@ class x86_page_ops(): def __nr_to_section(self, nr): root = self.SECTION_NR_TO_ROOT(nr) mem_section = gdb.parse_and_eval("mem_section") - return mem_section[root][nr & self.SECTION_ROOT_MASK] + return mem_section[root][nr % self.SECTIONS_PER_ROOT] def pfn_to_section_nr(self, pfn): return pfn >> self.PFN_SECTION_SHIFT @@ -233,11 +232,11 @@ class aarch64_page_ops(): self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS if str(constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER).isdigit(): - self.MAX_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER + self.MAX_PAGE_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER else: - self.MAX_ORDER = 10 + self.MAX_PAGE_ORDER = 10 - self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_ORDER) + self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_PAGE_ORDER) self.PFN_SECTION_SHIFT = self.SECTION_SIZE_BITS - self.PAGE_SHIFT self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT self.PAGES_PER_SECTION = 1 << self.PFN_SECTION_SHIFT @@ -249,7 +248,6 @@ class aarch64_page_ops(): self.SECTIONS_PER_ROOT = 1 self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT) - self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1 self.SUBSECTION_SHIFT = 21 self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT self.PFN_SUBSECTION_SHIFT = self.SUBSECTION_SHIFT - self.PAGE_SHIFT @@ -304,7 +302,7 @@ class aarch64_page_ops(): def __nr_to_section(self, nr): root = self.SECTION_NR_TO_ROOT(nr) mem_section = gdb.parse_and_eval("mem_section") - return mem_section[root][nr & self.SECTION_ROOT_MASK] + return mem_section[root][nr % self.SECTIONS_PER_ROOT] def pfn_to_section_nr(self, pfn): return pfn >> self.PFN_SECTION_SHIFT diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h index f2f54b038168..03ecf9086643 100644 --- a/tools/include/linux/compiler.h +++ b/tools/include/linux/compiler.h @@ -73,6 +73,11 @@ # define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b)) #endif +/* Enforce static storage duration. */ +#define ASSERT_STATIC_STORAGE(name) \ + static typeof(name) * const __always_unused \ + name##_storage_check = &(name) + /* * This returns a constant expression while determining if an argument is * a constant expression, most importantly without evaluating the argument. diff --git a/tools/include/linux/mm.h b/tools/include/linux/mm.h index 84b5954f66c3..d586a510e6e1 100644 --- a/tools/include/linux/mm.h +++ b/tools/include/linux/mm.h @@ -33,10 +33,6 @@ static inline phys_addr_t virt_to_phys(volatile void *address) return (phys_addr_t)address; } -static inline void totalram_pages_inc(void) -{ -} - static inline void totalram_pages_add(long count) { } diff --git a/tools/lib/mm/file_utils.c b/tools/lib/mm/file_utils.c new file mode 100644 index 000000000000..9b2237e9823e --- /dev/null +++ b/tools/lib/mm/file_utils.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +#include "file_utils.h" + +int read_file(const char *path, char *buf, size_t buflen) +{ + int fd, err; + ssize_t numread; + + fd = open(path, O_RDONLY); + if (fd == -1) + return -errno; + + numread = read(fd, buf, buflen - 1); + if (numread < 1) { + err = numread ? errno : ENODATA; + close(fd); + return -err; + } + + buf[numread] = '\0'; + close(fd); + + return 0; +} + +int write_file(const char *path, const char *buf, size_t buflen) +{ + int fd, saved_errno; + ssize_t numwritten; + + if (buflen < 2) + return -EINVAL; + + fd = open(path, O_WRONLY); + if (fd == -1) + return -errno; + + numwritten = write(fd, buf, buflen - 1); + saved_errno = errno; + close(fd); + + if (numwritten < 0) + return -saved_errno; + + if (numwritten != (ssize_t)(buflen - 1)) + return -EIO; + + return 0; +} + +int read_num(const char *path, unsigned long *num) +{ + unsigned long val; + int ret; + char buf[21]; + char *end; + + if (!num) + return -EINVAL; + + ret = read_file(path, buf, sizeof(buf)); + if (ret) + return ret; + + /* Reject signs and leading whitespace that are accepted by strtoul() */ + if (buf[0] < '0' || buf[0] > '9') + return -EINVAL; + + errno = 0; + val = strtoul(buf, &end, 10); + if (errno) + return -errno; + + /* Only allow a newline after the number */ + if (*end == '\n') + end++; + + if (*end != '\0') + return -EINVAL; + + *num = val; + return 0; +} + +int write_num(const char *path, unsigned long num) +{ + char buf[21]; + + sprintf(buf, "%lu", num); + return write_file(path, buf, strlen(buf) + 1); +} + +int write_num_ignore_einval(const char *path, unsigned long num) +{ + int ret; + + ret = write_num(path, num); + return ret == -EINVAL ? 0 : ret; +} diff --git a/tools/lib/mm/file_utils.h b/tools/lib/mm/file_utils.h new file mode 100644 index 000000000000..50daa82c2b2b --- /dev/null +++ b/tools/lib/mm/file_utils.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __MM_FILE_UTILS_H__ +#define __MM_FILE_UTILS_H__ + +#include + +int read_file(const char *path, char *buf, size_t buflen); +int write_file(const char *path, const char *buf, size_t buflen); +int read_num(const char *path, unsigned long *num); +int write_num(const char *path, unsigned long num); +int write_num_ignore_einval(const char *path, unsigned long num); + +#endif diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/lib/mm/hugepage_settings.c similarity index 76% rename from tools/testing/selftests/mm/hugepage_settings.c rename to tools/lib/mm/hugepage_settings.c index d7917dce3aba..77918677a9cd 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/lib/mm/hugepage_settings.c @@ -8,12 +8,18 @@ #include #include #include +#include -#include "vm_util.h" +#include "file_utils.h" #include "hugepage_settings.h" #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" #define MAX_SETTINGS_DEPTH 4 + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#endif + static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH]; static int settings_index; static struct thp_settings saved_settings; @@ -48,6 +54,11 @@ static const char * const shmem_enabled_strings[] = { NULL }; +static void print_file_access_error(const char *path, int ret) +{ + printf("# %s: %s (%d)\n", path, strerror(-ret), -ret); +} + int thp_read_string(const char *name, const char * const strings[]) { char path[PATH_MAX]; @@ -61,8 +72,9 @@ int thp_read_string(const char *name, const char * const strings[]) exit(EXIT_FAILURE); } - if (!read_file(path, buf, sizeof(buf))) { - perror(path); + ret = read_file(path, buf, sizeof(buf)); + if (ret) { + print_file_access_error(path, ret); exit(EXIT_FAILURE); } @@ -103,12 +115,17 @@ void thp_write_string(const char *name, const char *val) printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } - write_file(path, val, strlen(val) + 1); + ret = write_file(path, val, strlen(val) + 1); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } } unsigned long thp_read_num(const char *name) { char path[PATH_MAX]; + unsigned long num; int ret; ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); @@ -116,7 +133,13 @@ unsigned long thp_read_num(const char *name) printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } - return read_num(path); + ret = read_num(path, &num); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return num; } void thp_write_num(const char *name, unsigned long num) @@ -129,7 +152,11 @@ void thp_write_num(const char *name, unsigned long num) printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } - write_num(path, num); + ret = write_num(path, num); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } } void thp_read_settings(struct thp_settings *settings) @@ -157,8 +184,15 @@ void thp_read_settings(struct thp_settings *settings) .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"), .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"), }; - if (dev_queue_read_ahead_path[0]) - settings->read_ahead_kb = read_num(dev_queue_read_ahead_path); + if (dev_queue_read_ahead_path[0]) { + int ret = read_num(dev_queue_read_ahead_path, + &settings->read_ahead_kb); + + if (ret) { + print_file_access_error(dev_queue_read_ahead_path, ret); + exit(EXIT_FAILURE); + } + } for (i = 0; i < NR_ORDERS; i++) { if (!((1 << i) & orders)) { @@ -183,6 +217,13 @@ void thp_read_settings(struct thp_settings *settings) } } +/* A store to either sleep knob wakes khugepaged, so write only on change */ +static void thp_update_num(const char *name, unsigned long num) +{ + if (thp_read_num(name) != num) + thp_write_num(name, num); +} + void thp_write_settings(struct thp_settings *settings) { struct khugepaged_settings *khugepaged = &settings->khugepaged; @@ -198,18 +239,25 @@ void thp_write_settings(struct thp_settings *settings) shmem_enabled_strings[settings->shmem_enabled]); thp_write_num("use_zero_page", settings->use_zero_page); - thp_write_num("khugepaged/defrag", khugepaged->defrag); - thp_write_num("khugepaged/alloc_sleep_millisecs", - khugepaged->alloc_sleep_millisecs); - thp_write_num("khugepaged/scan_sleep_millisecs", - khugepaged->scan_sleep_millisecs); - thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none); - thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); - thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); - thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); + thp_update_num("khugepaged/defrag", khugepaged->defrag); + thp_update_num("khugepaged/alloc_sleep_millisecs", + khugepaged->alloc_sleep_millisecs); + thp_update_num("khugepaged/scan_sleep_millisecs", + khugepaged->scan_sleep_millisecs); + thp_update_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none); + thp_update_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); + thp_update_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); + thp_update_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); - if (dev_queue_read_ahead_path[0]) - write_num(dev_queue_read_ahead_path, settings->read_ahead_kb); + if (dev_queue_read_ahead_path[0]) { + int ret = write_num(dev_queue_read_ahead_path, + settings->read_ahead_kb); + + if (ret) { + print_file_access_error(dev_queue_read_ahead_path, ret); + exit(EXIT_FAILURE); + } + } for (i = 0; i < NR_ORDERS; i++) { if (!((1 << i) & orders)) @@ -230,6 +278,41 @@ void thp_write_settings(struct thp_settings *settings) } } +/* + * Wait for a full khugepaged scan pass that started after this call: the + * pass in progress may already have passed this mm, so full_scans has to + * advance twice. + * + * A store to scan_sleep_millisecs wakes the daemon, but one made while it + * is scanning rather than sleeping is lost, so keep storing until the pass + * lands. + * + * One wake is one pass only if pages_to_scan covers every mm on the list. + */ +bool khugepaged_full_pass(unsigned int timeout_s) +{ + unsigned long deadline_ms = timeout_s * 1000UL; + unsigned long elapsed_ms = 0, poll_ms = 10; + unsigned long sleep_ms; + int pass; + + sleep_ms = thp_read_num("khugepaged/scan_sleep_millisecs"); + for (pass = 0; pass < 2; pass++) { + unsigned long target = + thp_read_num("khugepaged/full_scans") + 1; + + while (thp_read_num("khugepaged/full_scans") < target) { + if (elapsed_ms >= deadline_ms) + return false; + thp_write_num("khugepaged/scan_sleep_millisecs", + sleep_ms); + usleep(poll_ms * 1000); + elapsed_ms += poll_ms; + } + } + return true; +} + struct thp_settings *thp_current_settings(void) { if (!settings_index) { @@ -307,8 +390,15 @@ static unsigned long __thp_supported_orders(bool is_shmem) } ret = read_file(path, buf, sizeof(buf)); - if (ret) - orders |= 1UL << i; + if (ret) { + if (ret != -ENOENT) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + continue; + } + + orders |= 1UL << i; } return orders; @@ -382,8 +472,7 @@ int detect_hugetlb_page_sizes(unsigned long sizes[], int max) if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) continue; sizes[count++] = kb * 1024; - ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", - kb); + printf("# [INFO] detected hugetlb page size: %zu KiB\n", kb); } closedir(dir); return count; @@ -425,28 +514,66 @@ static void hugetlb_sysfs_path(char *buf, size_t buflen, unsigned long hugetlb_nr_pages(unsigned long size) { char path[PATH_MAX]; + unsigned long nr; + int ret; hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - return read_num(path); + ret = read_num(path, &nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return nr; } void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) { char path[PATH_MAX]; + int ret; hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - write_num_ignore_einval(path, nr); + ret = write_num_ignore_einval(path, nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } } unsigned long hugetlb_free_pages(unsigned long size) { char path[PATH_MAX]; + unsigned long nr; + int ret; hugetlb_sysfs_path(path, sizeof(path), size, "free_hugepages"); - return read_num(path); + ret = read_num(path, &nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return nr; +} + +unsigned long hugetlb_nr_resv_pages(unsigned long size) +{ + char path[PATH_MAX]; + unsigned long nr; + int ret; + + hugetlb_sysfs_path(path, sizeof(path), size, "resv_hugepages"); + + ret = read_num(path, &nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return nr; } static bool __hugetlb_setup(unsigned long size, unsigned long nr) @@ -502,7 +629,8 @@ unsigned long hugetlb_setup(unsigned long nr, unsigned long sizes[], return 0; if (nr_enabled > max) { - ksft_print_msg("detected %d huge page sizes, will only test %d\n", nr_enabled, max); + printf("# detected %d huge page sizes, will only test %d\n", + nr_enabled, max); nr_enabled = max; } @@ -574,8 +702,10 @@ static void hugepage_restore_settings_atexit(void) static void hugepage_restore_settings_sighandler(int sig) { + (void)sig; + /* exit() will invoke the hugepage_restore_settings_atexit handler. */ - exit(KSFT_FAIL); + exit(EXIT_FAILURE); } void hugepage_save_settings(bool thp, bool hugetlb) diff --git a/tools/testing/selftests/mm/hugepage_settings.h b/tools/lib/mm/hugepage_settings.h similarity index 90% rename from tools/testing/selftests/mm/hugepage_settings.h rename to tools/lib/mm/hugepage_settings.h index 726c73c43c05..8f4581099b7a 100644 --- a/tools/testing/selftests/mm/hugepage_settings.h +++ b/tools/lib/mm/hugepage_settings.h @@ -83,10 +83,21 @@ static inline void thp_save_settings(void) hugepage_save_settings(/* thp = */ true, /* hugetlb = */ false); } +bool khugepaged_full_pass(unsigned int timeout_s); + void thp_set_read_ahead_path(char *path); unsigned long thp_supported_orders(void); unsigned long thp_shmem_supported_orders(void); +/* + * The per-order shmem_enabled attribute is created for the orders the page + * cache can hold, not just for shmem, so it answers for regular files too. + */ +static inline unsigned long thp_file_supported_orders(void) +{ + return thp_shmem_supported_orders(); +} + bool thp_available(void); bool thp_is_enabled(void); @@ -98,6 +109,7 @@ unsigned long default_huge_page_size(void); unsigned long hugetlb_nr_pages(unsigned long size); void hugetlb_set_nr_pages(unsigned long size, unsigned long nr); unsigned long hugetlb_free_pages(unsigned long size); +unsigned long hugetlb_nr_resv_pages(unsigned long size); static inline void hugetlb_save_settings(void) { diff --git a/tools/mm/.gitignore b/tools/mm/.gitignore index 1446a659e540..154d740be02e 100644 --- a/tools/mm/.gitignore +++ b/tools/mm/.gitignore @@ -3,3 +3,4 @@ slabinfo page-types page_owner_sort thp_swap_allocator_test +gup_bench diff --git a/tools/mm/Makefile b/tools/mm/Makefile index 858186a6eefd..f20a32d8cc22 100644 --- a/tools/mm/Makefile +++ b/tools/mm/Makefile @@ -3,13 +3,15 @@ # include ../scripts/Makefile.include -BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test +BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter +BUILD_TARGETS += thp_swap_allocator_test gup_bench INSTALL_TARGETS = $(BUILD_TARGETS) thpmaps LIB_DIR = ../lib/api LIBS = $(LIB_DIR)/libapi.a +GUP_BENCH_OBJS = gup_bench.c ../lib/mm/hugepage_settings.c ../lib/mm/file_utils.c -CFLAGS += -Wall -Wextra -I../lib/ -pthread +CFLAGS += -Wall -Wextra -I../lib/ -I../.. -pthread LDFLAGS += $(LIBS) -pthread all: $(BUILD_TARGETS) @@ -22,8 +24,11 @@ $(LIBS): %: %.c $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) +gup_bench: $(GUP_BENCH_OBJS) $(LIBS) + $(CC) $(CFLAGS) -o $@ $(GUP_BENCH_OBJS) $(LDFLAGS) + clean: - $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test + $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test gup_bench make -C $(LIB_DIR) clean sbindir ?= /usr/sbin diff --git a/tools/testing/selftests/mm/gup_test.c b/tools/mm/gup_bench.c similarity index 50% rename from tools/testing/selftests/mm/gup_test.c rename to tools/mm/gup_bench.c index 3f841a96f870..ff6e46654681 100644 --- a/tools/testing/selftests/mm/gup_test.c +++ b/tools/mm/gup_bench.c @@ -10,11 +10,10 @@ #include #include #include -#include +#include +#include #include -#include "kselftest.h" -#include "vm_util.h" -#include "hugepage_settings.h" +#include #define MB (1UL << 20) @@ -38,12 +37,6 @@ static char *cmd_to_str(unsigned long cmd) return "PIN_FAST_BENCHMARK"; case PIN_LONGTERM_BENCHMARK: return "PIN_LONGTERM_BENCHMARK"; - case GUP_BASIC_TEST: - return "GUP_BASIC_TEST"; - case PIN_BASIC_TEST: - return "PIN_BASIC_TEST"; - case DUMP_USER_PAGES_TEST: - return "DUMP_USER_PAGES_TEST"; } return "Unknown command"; } @@ -53,39 +46,29 @@ void *gup_thread(void *data) struct gup_test gup = *(struct gup_test *)data; int i, status; - /* Only report timing information on the *_BENCHMARK commands: */ - if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) || - (cmd == PIN_LONGTERM_BENCHMARK)) { - for (i = 0; i < repeats; i++) { - gup.size = size; - status = ioctl(gup_fd, cmd, &gup); - if (status) - break; - - pthread_mutex_lock(&print_mutex); - ksft_print_msg("%s: Time: get:%lld put:%lld us", - cmd_to_str(cmd), gup.get_delta_usec, - gup.put_delta_usec); - if (gup.size != size) - ksft_print_msg(", truncated (size: %lld)", gup.size); - ksft_print_msg("\n"); - pthread_mutex_unlock(&print_mutex); - } - } else { + for (i = 0; i < repeats; i++) { gup.size = size; status = ioctl(gup_fd, cmd, &gup); - if (status) - goto return_; + if (status) { + int err = errno; + + pthread_mutex_lock(&print_mutex); + fprintf(stderr, "%s ioctl failed: %s\n", cmd_to_str(cmd), + strerror(err)); + pthread_mutex_unlock(&print_mutex); + return data; + } pthread_mutex_lock(&print_mutex); - ksft_print_msg("%s: done\n", cmd_to_str(cmd)); + printf("%s: Time: get:%lld put:%lld us", + cmd_to_str(cmd), gup.get_delta_usec, + gup.put_delta_usec); if (gup.size != size) - ksft_print_msg("Truncated (size: %lld)\n", gup.size); + printf(", truncated (size: %lld)", gup.size); + printf("\n"); pthread_mutex_unlock(&print_mutex); } -return_: - ksft_test_result(!status, "ioctl status %d\n", status); return NULL; } @@ -93,38 +76,21 @@ int main(int argc, char **argv) { struct gup_test gup = { 0 }; int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; - int flags = MAP_PRIVATE; + int flags = MAP_PRIVATE, started_threads = 0, exit_status = 1; char *file = "/dev/zero"; - bool hugetlb = false; + bool hugetlb = false, thread_error = false; + void *thread_result; pthread_t *tid; char *p; - while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) { + while ((opt = getopt(argc, argv, "m:r:n:F:f:aj:tTLuwWSH")) != -1) { switch (opt) { case 'a': cmd = PIN_FAST_BENCHMARK; break; - case 'b': - cmd = PIN_BASIC_TEST; - break; case 'L': cmd = PIN_LONGTERM_BENCHMARK; break; - case 'c': - cmd = DUMP_USER_PAGES_TEST; - /* - * Dump page 0 (index 1). May be overridden later, by - * user's non-option arguments. - * - * .which_pages is zero-based, so that zero can mean "do - * nothing". - */ - gup.which_pages[0] = 1; - break; - case 'p': - /* works only with DUMP_USER_PAGES_TEST */ - gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN; - break; case 'F': /* strtol, so you can pass flags in hex form */ gup.gup_flags = strtol(optarg, 0, 0); @@ -141,7 +107,7 @@ int main(int argc, char **argv) case 'n': nr_pages = atoi(optarg); if (nr_pages < 0) - nr_pages = size / psize(); + nr_pages = size / getpagesize(); break; case 't': thp = 1; @@ -149,9 +115,6 @@ int main(int argc, char **argv) case 'T': thp = 0; break; - case 'U': - cmd = GUP_BASIC_TEST; - break; case 'u': cmd = GUP_FAST_BENCHMARK; break; @@ -173,52 +136,41 @@ int main(int argc, char **argv) hugetlb = true; break; default: - ksft_exit_fail_msg("Wrong argument\n"); + fprintf(stderr, "Wrong argument\n"); + exit(1); } } - if (optind < argc) { - int extra_arg_count = 0; - /* - * For example: - * - * ./gup_test -c 0 1 0x1001 - * - * ...to dump pages 0, 1, and 4097 - */ - - while ((optind < argc) && - (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) { - /* - * Do the 1-based indexing here, so that the user can - * use normal 0-based indexing on the command line. - */ - long page_index = strtol(argv[optind], 0, 0) + 1; - - gup.which_pages[extra_arg_count] = page_index; - extra_arg_count++; - optind++; - } + if (optind != argc) { + fprintf(stderr, "Unexpected argument '%s'\n", argv[optind]); + exit(1); } - ksft_print_header(); + if (geteuid()) { + fprintf(stderr, "Please run this test as root\n"); + exit(1); + } if (hugetlb) { unsigned long hp_size = default_huge_page_size(); - if (!hp_size) - ksft_exit_skip("HugeTLB is unavailable\n"); + if (!hp_size) { + fprintf(stderr, "Could not determine huge page size\n"); + return 1; + } size = (size + hp_size - 1) & ~(hp_size - 1); - if (!hugetlb_setup_default(size / hp_size)) - ksft_exit_skip("Not enough huge pages\n"); + if (!hugetlb_setup_default(size / hp_size)) { + fprintf(stderr, "Not enough huge pages\n"); + return 1; + } } - ksft_set_plan(nthreads); - filed = open(file, O_RDWR|O_CREAT, 0664); - if (filed < 0) - ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno)); + if (filed < 0) { + fprintf(stderr, "Unable to open %s: %s\n", file, strerror(errno)); + return 1; + } gup.nr_pages_per_call = nr_pages; if (write) @@ -227,26 +179,24 @@ int main(int argc, char **argv) gup_fd = open(GUP_TEST_FILE, O_RDWR); if (gup_fd == -1) { switch (errno) { - case EACCES: - if (getuid()) - ksft_print_msg("Please run this test as root\n"); - break; case ENOENT: if (opendir("/sys/kernel/debug") == NULL) - ksft_print_msg("mount debugfs at /sys/kernel/debug\n"); - ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n"); + fprintf(stderr, "mount debugfs at /sys/kernel/debug\n"); + fprintf(stderr, "check if CONFIG_GUP_TEST is enabled in kernel config\n"); break; default: - ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno)); + fprintf(stderr, "failed to open %s: %s\n", GUP_TEST_FILE, + strerror(errno)); break; } - ksft_test_result_skip("Please run this test as root\n"); - ksft_exit_pass(); + goto err_close_filed; } p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0); - if (p == MAP_FAILED) - ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + if (p == MAP_FAILED) { + fprintf(stderr, "mmap: %s\n", strerror(errno)); + goto err_close_gup_fd; + } gup.addr = (unsigned long)p; if (thp == 1) @@ -255,21 +205,43 @@ int main(int argc, char **argv) madvise(p, size, MADV_NOHUGEPAGE); /* Fault them in here, from user space. */ - for (; (unsigned long)p < gup.addr + size; p += psize()) + for (; (unsigned long)p < gup.addr + size; p += getpagesize()) p[0] = 0; tid = malloc(sizeof(pthread_t) * nthreads); - assert(tid); + if (!tid) { + fprintf(stderr, "Failed to allocate %d threads: %s\n", + nthreads, strerror(errno)); + goto err_unmap; + } + for (i = 0; i < nthreads; i++) { ret = pthread_create(&tid[i], NULL, gup_thread, &gup); - assert(ret == 0); + if (ret) { + fprintf(stderr, "pthread_create failed: %s\n", strerror(ret)); + thread_error = true; + break; + } + started_threads++; } - for (i = 0; i < nthreads; i++) { - ret = pthread_join(tid[i], NULL); - assert(ret == 0); + for (i = 0; i < started_threads; i++) { + ret = pthread_join(tid[i], &thread_result); + if (ret) { + fprintf(stderr, "pthread_join failed: %s\n", strerror(ret)); + thread_error = true; + } else if (thread_result) + thread_error = true; } free(tid); + if (!thread_error) + exit_status = 0; - ksft_exit_pass(); +err_unmap: + munmap((void *)gup.addr, size); +err_close_gup_fd: + close(gup_fd); +err_close_filed: + close(filed); + return exit_status; } diff --git a/tools/mm/page-types.c b/tools/mm/page-types.c index 7fc5a8be5997..47e4781c5fc3 100644 --- a/tools/mm/page-types.c +++ b/tools/mm/page-types.c @@ -73,7 +73,6 @@ #define KPF_RESERVED 32 #define KPF_MLOCKED 33 #define KPF_OWNER_2 34 -#define KPF_PRIVATE 35 #define KPF_PRIVATE_2 36 #define KPF_OWNER_PRIVATE 37 #define KPF_ARCH 38 @@ -131,7 +130,6 @@ static const char * const page_flag_names[] = { [KPF_RESERVED] = "r:reserved", [KPF_MLOCKED] = "m:mlocked", [KPF_OWNER_2] = "d:owner_2", - [KPF_PRIVATE] = "P:private", [KPF_PRIVATE_2] = "p:private_2", [KPF_OWNER_PRIVATE] = "O:owner_private", [KPF_ARCH] = "h:arch", diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index 22b3b500d33a..6c2ac5d9dc4b 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -25,11 +25,13 @@ #include #define TASK_COMM_LEN 16 +#define MODULE_NAME_LEN 64 struct block_list { char *txt; char *comm; // task command name char *stacktrace; + char *module; // kernel module name __u64 ts_nsec; int len; int num; @@ -41,7 +43,8 @@ struct block_list { enum FILTER_BIT { FILTER_PID = 1<<1, FILTER_TGID = 1<<2, - FILTER_COMM = 1<<3 + FILTER_COMM = 1<<3, + FILTER_MODULE = 1<<4 }; enum FILTER_RESULT { @@ -55,7 +58,8 @@ enum CULL_BIT { CULL_TGID = 1<<2, CULL_COMM = 1<<3, CULL_STACKTRACE = 1<<4, - CULL_ALLOCATOR = 1<<5 + CULL_ALLOCATOR = 1<<5, + CULL_MODULE = 1<<6 }; enum ALLOCATOR_BIT { ALLOCATOR_CMA = 1<<1, @@ -65,7 +69,8 @@ enum ALLOCATOR_BIT { }; enum ARG_TYPE { ARG_TXT, ARG_COMM, ARG_STACKTRACE, ARG_ALLOC_TS, ARG_CULL_TIME, - ARG_PAGE_NUM, ARG_PID, ARG_TGID, ARG_UNKNOWN, ARG_ALLOCATOR + ARG_PAGE_NUM, ARG_PID, ARG_TGID, ARG_UNKNOWN, ARG_ALLOCATOR, + ARG_MODULE }; enum SORT_ORDER { SORT_ASC = 1, @@ -79,15 +84,18 @@ enum COMP_FLAG { COMP_STACK = 1<<3, COMP_NUM = 1<<4, COMP_TGID = 1<<5, - COMP_COMM = 1<<6 + COMP_COMM = 1<<6, + COMP_MODULE = 1<<7 }; struct filter_condition { pid_t *pids; pid_t *tgids; char **comms; + char **modules; int pids_size; int tgids_size; int comms_size; + int modules_size; }; struct sort_condition { int (**cmps)(const void *, const void *); @@ -101,6 +109,7 @@ static regex_t pid_pattern; static regex_t tgid_pattern; static regex_t comm_pattern; static regex_t ts_nsec_pattern; +static regex_t module_pattern; static struct block_list *list; static int list_size; static int max_size; @@ -184,6 +193,13 @@ static int compare_comm(const void *p1, const void *p2) return strcmp(l1->comm, l2->comm); } +static int compare_module(const void *p1, const void *p2) +{ + const struct block_list *l1 = p1, *l2 = p2; + + return strcmp(l1->module, l2->module); +} + static int compare_ts(const void *p1, const void *p2) { const struct block_list *l1 = p1, *l2 = p2; @@ -207,6 +223,8 @@ static int compare_cull_condition(const void *p1, const void *p2) return compare_tgid(p1, p2); if ((cull & CULL_COMM) && compare_comm(p1, p2)) return compare_comm(p1, p2); + if ((cull & CULL_MODULE) && compare_module(p1, p2)) + return compare_module(p1, p2); if ((cull & CULL_ALLOCATOR) && compare_allocator(p1, p2)) return compare_allocator(p1, p2); return 0; @@ -411,9 +429,33 @@ static char *get_comm(char *buf) return comm_str; } +static char *get_module(char *buf) +{ + char *module_str = malloc(MODULE_NAME_LEN); + regmatch_t pmatch[2]; + int val_len; + + if (!module_str) + return NULL; + memset(module_str, 0, MODULE_NAME_LEN); + if (regexec(&module_pattern, buf, 2, pmatch, REG_NOTBOL) != 0 || pmatch[1].rm_so == -1) { + strcpy(module_str, "vmlinux"); + return module_str; + } + + val_len = pmatch[1].rm_eo - pmatch[1].rm_so; + if ((size_t)val_len >= MODULE_NAME_LEN) + val_len = MODULE_NAME_LEN - 1; + memcpy(module_str, buf + pmatch[1].rm_so, val_len); + module_str[val_len] = '\0'; + + return module_str; +} + static void free_block_list(struct block_list *block) { free(block->comm); + free(block->module); free(block->txt); } @@ -433,6 +475,8 @@ static int get_arg_type(const char *arg) return ARG_ALLOC_TS; else if (!strcmp(arg, "allocator") || !strcmp(arg, "ator")) return ARG_ALLOCATOR; + else if (!strcmp(arg, "module") || !strcmp(arg, "mod")) + return ARG_MODULE; else { return ARG_UNKNOWN; } @@ -483,25 +527,36 @@ static bool match_str_list(const char *str, char **list, int list_size) static enum FILTER_RESULT filter_record(char *buf) { - char *comm; + char *comm, *module; if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size)) return FILTER_SKIP; if ((filter & FILTER_TGID) && !match_num_list(get_tgid(buf), fc.tgids, fc.tgids_size)) return FILTER_SKIP; - if (!(filter & FILTER_COMM)) + if (!(filter & (FILTER_COMM | FILTER_MODULE))) return FILTER_MATCH; - comm = get_comm(buf); - if (!comm) - return FILTER_ERROR; - - if (!match_str_list(comm, fc.comms, fc.comms_size)) { + if (filter & FILTER_COMM) { + comm = get_comm(buf); + if (!comm) + return FILTER_ERROR; + if (!match_str_list(comm, fc.comms, fc.comms_size)) { + free(comm); + return FILTER_SKIP; + } free(comm); - return FILTER_SKIP; } - free(comm); + if (filter & FILTER_MODULE) { + module = get_module(buf); + if (!module) + return FILTER_ERROR; + if (!match_str_list(module, fc.modules, fc.modules_size)) { + free(module); + return FILTER_SKIP; + } + free(module); + } return FILTER_MATCH; } @@ -547,6 +602,12 @@ static bool add_list(char *buf, int len, char *ext_buf) list[list_size].stacktrace++; list[list_size].ts_nsec = get_ts_nsec(buf); list[list_size].allocator = get_allocator(buf, ext_buf); + list[list_size].module = get_module(buf); + if (!list[list_size].module) { + fprintf(stderr, "Out of memory\n"); + free_block_list(&list[list_size]); + return false; + } list_size++; if (list_size % 1000 == 0) { printf("loaded %d\r", list_size); @@ -573,6 +634,8 @@ static bool parse_cull_args(const char *arg_str) cull |= CULL_STACKTRACE; else if (arg_type == ARG_ALLOCATOR) cull |= CULL_ALLOCATOR; + else if (arg_type == ARG_MODULE) + cull |= CULL_MODULE; else { free_explode(args, size); return false; @@ -635,6 +698,8 @@ static bool parse_sort_args(const char *arg_str) sc.cmps[i] = compare_txt; else if (arg_type == ARG_ALLOCATOR) sc.cmps[i] = compare_allocator; + else if (arg_type == ARG_MODULE) + sc.cmps[i] = compare_module; else { free_explode(args, size); sc.size = 0; @@ -691,7 +756,8 @@ static void usage(void) "-p\t\t\tSort by pid.\n" "-P\t\t\tSort by tgid.\n" "-s\t\t\tSort by the stacktrace.\n" - "-t\t\t\tSort by number of times record is seen (default).\n\n" + "-t\t\t\tSort by number of times record is seen (default).\n" + "-M\t\t\tSort by module name.\n\n" "--pid \t\tSelect by pid. This selects the information" " of\n\t\t\tblocks whose process ID numbers appear in .\n" "--tgid \tSelect by tgid. This selects the information" @@ -700,11 +766,21 @@ static void usage(void) "--name \tSelect by command name. This selects the" " information\n\t\t\tof blocks whose command name appears in" " .\n" - "--cull \t\tCull by user-defined rules. is a " - "single\n\t\t\targument in the form of a comma-separated list " - "with some\n\t\t\tcommon fields predefined (pid, tgid, comm, " - "stacktrace, allocator)\n" + "--module \tSelect by module name. This selects the information\n" + "\t\t\tof blocks whose module name appears in .\n" + "--cull \t\tCull by user-defined rules. is a single\n" + "\t\t\targument in the form of a comma-separated list with some\n" + "\t\t\tcommon fields predefined (pid, tgid, comm, stacktrace, allocator, module)\n" "--sort \t\tSpecify sort order as: [+|-]key[,[+|-]key[,...]]\n" + "\t\t\tAvailable keys:\n" + "\t\t\t pid(p), tgid(tg), name(n), stacktrace(st),\n" + "\t\t\t txt(T), alloc_ts(at), allocator(ator), module(mod)\n" + "\t\t\tThe \"+\" is optional since default direction is\n" + "\t\t\tincreasing numerical or lexicographic order.\n" + "\t\t\tMixed use of abbreviated and complete-form is allowed.\n" + "\t\t\tExamples:\n" + "\t\t\t --sort=n,+pid,-tgid\n" + "\t\t\t --sort=mod,at\n" ); } @@ -721,13 +797,14 @@ int main(int argc, char **argv) { "name", required_argument, NULL, 3 }, { "cull", required_argument, NULL, 4 }, { "sort", required_argument, NULL, 5 }, + { "module", required_argument, NULL, 6 }, { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0}, }; compare_flag = COMP_NO_FLAG; - while ((opt = getopt_long(argc, argv, "admnpstPh", longopts, NULL)) != -1) + while ((opt = getopt_long(argc, argv, "admnpstPMh", longopts, NULL)) != -1) switch (opt) { case 'a': compare_flag |= COMP_ALLOC; @@ -753,6 +830,9 @@ int main(int argc, char **argv) case 'n': compare_flag |= COMP_COMM; break; + case 'M': + compare_flag |= COMP_MODULE; + break; case 'h': usage(); exit(0); @@ -792,6 +872,10 @@ int main(int argc, char **argv) exit(1); } break; + case 6: + filter = filter | FILTER_MODULE; + fc.modules = explode(',', optarg, &fc.modules_size); + break; default: usage(); exit(1); @@ -821,6 +905,9 @@ int main(int argc, char **argv) set_single_cmp(compare_stacktrace, SORT_ASC); break; case COMP_NO_FLAG: + if (sc.size > 0) + break; + /* fallthrough */ case COMP_NUM: set_single_cmp(compare_num, SORT_DESC); break; @@ -830,6 +917,9 @@ int main(int argc, char **argv) case COMP_COMM: set_single_cmp(compare_comm, SORT_ASC); break; + case COMP_MODULE: + set_single_cmp(compare_module, SORT_ASC); + break; default: usage(); exit(1); @@ -852,6 +942,8 @@ int main(int argc, char **argv) goto out_comm; if (!check_regcomp(&ts_nsec_pattern, "ts\\s*([0-9]*)\\s*ns")) goto out_ts; + if (!check_regcomp(&module_pattern, "\\+0x[0-9a-f]+/0x[0-9a-f]+\\s*\\[([a-zA-Z0-9_-]+)\\]")) + goto out_module; fstat(fileno(fin), &st); max_size = st.st_size / 100; /* hack ... */ @@ -917,6 +1009,8 @@ int main(int argc, char **argv) fprintf(fout, ", TGID %d", list[i].tgid); if (cull & CULL_COMM || filter & FILTER_COMM) fprintf(fout, ", task_comm_name: %s", list[i].comm); + if (cull & CULL_MODULE || filter & FILTER_MODULE) + fprintf(fout, ", module: %s", list[i].module); if (cull & CULL_ALLOCATOR) { fprintf(fout, ", "); print_allocator(fout, list[i].allocator); @@ -937,6 +1031,8 @@ out_free: free_block_list(&list[i]); free(list); } +out_module: + regfree(&module_pattern); out_ts: regfree(&ts_nsec_pattern); out_comm: diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 9376d5704a47..6194484d4d9a 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -32,8 +32,6 @@ #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ -#define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ -#define PF_KSWAPD 0x00020000 /* I am kswapd */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_EXITING 0x00000004 #define CLOCK_MONOTONIC 1 diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c index 945144e98507..6fcba5b5870b 100644 --- a/tools/testing/radix-tree/idr-test.c +++ b/tools/testing/radix-tree/idr-test.c @@ -460,9 +460,11 @@ void ida_dump(struct ida *); */ void ida_check_nomem(void) { - DEFINE_IDA(ida); + struct ida ida; int id; + ida_init(&ida); + id = ida_alloc_min(&ida, 256, GFP_NOWAIT); IDA_BUG_ON(&ida, id != -ENOMEM); id = ida_alloc_min(&ida, 1UL << 30, GFP_NOWAIT); @@ -475,9 +477,11 @@ void ida_check_nomem(void) */ void ida_check_conv_user(void) { - DEFINE_IDA(ida); + struct ida ida; unsigned long i; + ida_init(&ida); + for (i = 0; i < 1000000; i++) { int id = ida_alloc(&ida, GFP_NOWAIT); if (id == -ENOMEM) { @@ -496,11 +500,13 @@ void ida_check_conv_user(void) void ida_check_random(void) { - DEFINE_IDA(ida); + struct ida ida; DECLARE_BITMAP(bitmap, 2048); unsigned int i; time_t s = time(NULL); + ida_init(&ida); + repeat: memset(bitmap, 0, sizeof(bitmap)); for (i = 0; i < 100000; i++) { @@ -522,9 +528,11 @@ void ida_check_random(void) void ida_alloc_free_test(void) { - DEFINE_IDA(ida); + struct ida ida; unsigned long i; + ida_init(&ida); + for (i = 0; i < 10000; i++) assert(ida_alloc_max(&ida, 20000, GFP_KERNEL) == i); assert(ida_alloc_range(&ida, 5, 30, GFP_KERNEL) < 0); @@ -576,10 +584,12 @@ static void *ida_leak_fn(void *arg) void ida_thread_tests(void) { - DEFINE_IDA(ida); + struct ida ida; pthread_t threads[20]; int i; + ida_init(&ida); + for (i = 0; i < ARRAY_SIZE(threads); i++) if (pthread_create(&threads[i], NULL, ida_random_fn, NULL)) { perror("creating ida thread"); diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index d967e76a3c06..bfe4b8626c42 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -36022,10 +36022,12 @@ static noinline void __init check_erase_rebalance(struct maple_tree *mt) static noinline void __init check_mtree_dup(struct maple_tree *mt) { - DEFINE_MTREE(new); + struct maple_tree new; int i, j, ret, count = 0; unsigned int rand_seed = 17, rand; + mt_init(&new); + /* store a value at [0, 0] */ mt_init_flags(mt, 0); mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); @@ -36319,7 +36321,9 @@ static inline int check_vma_modification(struct maple_tree *mt) void farmer_tests(void) { struct maple_node *node; - DEFINE_MTREE(tree); + struct maple_tree tree; + + mt_init(&tree); mt_dump(&tree, mt_dump_dec); @@ -36432,9 +36436,11 @@ static unsigned long get_last_index(struct ma_state *mas) static void test_spanning_store_regression(void) { unsigned long from = 0, to = 0; - DEFINE_MTREE(tree); + struct maple_tree tree; MA_STATE(mas, &tree, 0, 0); + mt_init(&tree); + /* * Build a 3-level tree. We require a parent node below the root node * and 2 leaf nodes under it, so we can span the entirety of the right diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 8df54b59513a..d50acc1b83ac 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -346,7 +346,16 @@ static int attempt_writeback(const char *cgroup, void *arg) * it can't writeback to swap. */ ret = cg_write_numeric(cgroup, "memory.reclaim", memsize); - if (!wb_enabled) + + /* + * When writeback is enabled, memory.reclaim may still fail to reclaim + * the requested amount of memory due to a slow swap device. + * Ignore -EAGAIN here. The caller determines pass/fail based on the + * zswap writeback counter. + */ + if (wb_enabled && ret == -EAGAIN) + ret = 0; + else if (!wb_enabled) ret = (ret == -EAGAIN) ? 0 : -1; out: @@ -409,6 +418,8 @@ static int test_zswap_writeback(const char *root, bool wb) * Thus, the parent's setting shall be what's in effect. */ if (cg_write(test_group, "memory.zswap.max", "max")) goto out; + if (cg_read_strcmp_wait(test_group, "cgroup.events", "populated 0\n")) + goto out; if (cg_write(test_group, "cgroup.subtree_control", "+memory")) goto out; @@ -652,11 +663,14 @@ static int test_no_kmem_bypass(const char *root) break; /* If memory was pushed to zswap, verify it belongs to memcg */ if (stored_pages > stored_pages_threshold) { - int zswapped = cg_read_key_long(test_group, "memory.stat", "zswapped "); - int delta = stored_pages * page_size - zswapped; - int result_ok = delta < stored_pages * page_size / 4; + long zswapped = cg_read_key_long( + test_group, "memory.stat", "zswapped "); + long long delta = + (long long)stored_pages * page_size - zswapped; + long long max_delta = + (long long)stored_pages * page_size / 4; - ret = result_ok ? KSFT_PASS : KSFT_FAIL; + ret = (delta < max_delta) ? KSFT_PASS : KSFT_FAIL; break; } } diff --git a/tools/testing/selftests/damon/.gitignore b/tools/testing/selftests/damon/.gitignore index 2f0297657c81..226bd7c4e072 100644 --- a/tools/testing/selftests/damon/.gitignore +++ b/tools/testing/selftests/damon/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only access_memory access_memory_even +damon_dump_output diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index e6a2265d721e..e7095c336524 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -321,8 +321,10 @@ class DamosFilters: filters = None scheme = None # owner scheme - def __init__(self, name, filters=[]): + def __init__(self, name, filters=None): self.name = name + if filters is None: + filters = [] self.filters = filters for idx, filter_ in enumerate(self.filters): filter_.idx = idx @@ -368,7 +370,9 @@ class DamosDests: dests = None scheme = None # owner scheme - def __init__(self, dests=[]): + def __init__(self, dests=None): + if dests is None: + dests = [] self.dests = dests for idx, dest in enumerate(self.dests): dest.idx = idx @@ -426,15 +430,21 @@ class Damos: stats = None tried_regions = None - def __init__(self, action='stat', access_pattern=DamosAccessPattern(), - quota=DamosQuota(), watermarks=DamosWatermarks(), - core_filters=[], ops_filters=[], filters=[], target_nid=0, - dests=DamosDests(), apply_interval_us=0): + def __init__(self, action='stat', access_pattern=None, quota=None, + watermarks=None, core_filters=None, ops_filters=None, + filters=None, target_nid=0, dests=None, + apply_interval_us=0): self.action = action + if access_pattern is None: + access_pattern = DamosAccessPattern() self.access_pattern = access_pattern self.access_pattern.scheme = self + if quota is None: + quota = DamosQuota() self.quota = quota self.quota.scheme = self + if watermarks is None: + watermarks = DamosWatermarks() self.watermarks = watermarks self.watermarks.scheme = self @@ -448,6 +458,8 @@ class Damos: self.filters.scheme = self self.target_nid = target_nid + if dests is None: + dests = DamosDests() self.dests = dests self.dests.scheme = self @@ -558,6 +570,117 @@ class IntervalsGoal: return err return None +class DamonFilter: + type_ = None + matching = None + allow = None + filters = None + path = None + idx = None + + def __init__(self, type_='anon', matching=False, allow=False, path=None): + self.type_ = type_ + self.matching = matching + self.allow = allow + self.path = path + + def sysfs_dir(self): + return os.path.join(self.filters.sysfs_dir(), '%d' % self.idx) + + def stage(self): + err = write_file(os.path.join(self.sysfs_dir(), 'type'), self.type_) + if err is not None: + return err + err = write_file(os.path.join(self.sysfs_dir(), 'matching'), + 'Y' if self.matching else 'N') + if err is not None: + return err + err = write_file(os.path.join(self.sysfs_dir(), 'allow'), + 'Y' if self.allow else 'N') + if err is not None: + return err + if self.type_ == 'memcg': + err = write_file(os.path.join(self.sysfs_dir(), 'path'), self.path) + if err is not None: + return err + return None + +class DamonFilters: + filters = None + probe = None + + def __init__(self, filters=None): + if filters is None: + filters = [] + self.filters = filters + for idx, filter in enumerate(self.filters): + filter.filters = self + filter.idx = idx + + def sysfs_dir(self): + return os.path.join(self.probe.sysfs_dir(), 'filters') + + def stage(self): + err = write_file( + os.path.join(self.sysfs_dir(), 'nr_filters'), + len(self.filters)) + if err is not None: + return err + for filter in self.filters: + err = filter.stage() + if err is not None: + return err + return None + +class DamonProbe: + weight = None + filters = None + probes = None + idx = None + + def __init__(self, weight=0, filters=None): + self.weight = weight + if filters is None: + filters = DamonFilters() + self.filters = filters + self.filters.probe = self + + def sysfs_dir(self): + return os.path.join(self.probes.sysfs_dir(), '%d' % self.idx) + + def stage(self): + err = write_file( + os.path.join(self.sysfs_dir(), 'weight'), '%d' % self.weight) + if err is not None: + return err + return self.filters.stage() + +class DamonProbes: + probes = None + attrs = None + + def __init__(self, probes=None): + if probes is None: + probes = [] + self.probes = probes + for idx, probe in enumerate(self.probes): + probe.probes = self + probe.idx = idx + + def sysfs_dir(self): + return os.path.join(self.attrs.sysfs_dir(), 'probes') + + def stage(self): + err = write_file(os.path.join(self.sysfs_dir(), 'nr_probes'), + len(self.probes)) + if err is not None: + return err + for probe in self.probes: + err = probe.stage() + if err is not None: + return err + return None + class DamonAttrs: sample_us = None aggr_us = None @@ -565,18 +688,25 @@ class DamonAttrs: update_us = None min_nr_regions = None max_nr_regions = None + probes = None context = None def __init__(self, sample_us=5000, aggr_us=100000, - intervals_goal=IntervalsGoal(), update_us=1000000, - min_nr_regions=10, max_nr_regions=1000): + intervals_goal=None, update_us=1000000, min_nr_regions=10, + max_nr_regions=1000, probes=None): self.sample_us = sample_us self.aggr_us = aggr_us + if intervals_goal is None: + intervals_goal = IntervalsGoal() self.intervals_goal = intervals_goal self.intervals_goal.attrs = self self.update_us = update_us self.min_nr_regions = min_nr_regions self.max_nr_regions = max_nr_regions + if probes is None: + probes = DamonProbes() + self.probes = probes + self.probes.attrs = self def interval_sysfs_dir(self): return os.path.join(self.context.sysfs_dir(), 'monitoring_attrs', @@ -586,6 +716,9 @@ class DamonAttrs: return os.path.join(self.context.sysfs_dir(), 'monitoring_attrs', 'nr_regions') + def sysfs_dir(self): + return os.path.join(self.context.sysfs_dir(), 'monitoring_attrs') + def stage(self): err = write_file(os.path.join(self.interval_sysfs_dir(), 'sample_us'), self.sample_us) @@ -615,6 +748,10 @@ class DamonAttrs: if err is not None: return err + err = self.probes.stage() + if err is not None: + return err + class DamonCtx: ops = None monitoring_attrs = None @@ -703,7 +840,9 @@ class Kdamond: idx = None # index of this kdamond between siblings kdamonds = None # parent - def __init__(self, contexts=[], refresh_ms=None): + def __init__(self, contexts=None, refresh_ms=None): + if contexts is None: + contexts = [] self.contexts = contexts self.refresh_ms = refresh_ms for idx, context in enumerate(self.contexts): @@ -853,7 +992,9 @@ class Kdamond: class Kdamonds: kdamonds = [] - def __init__(self, kdamonds=[]): + def __init__(self, kdamonds=None): + if kdamonds is None: + kdamonds = [] self.kdamonds = kdamonds for idx, kdamond in enumerate(self.kdamonds): kdamond.idx = idx diff --git a/tools/testing/selftests/damon/damon_nr_regions.py b/tools/testing/selftests/damon/damon_nr_regions.py index 58f3291fed12..e55239813c65 100755 --- a/tools/testing/selftests/damon/damon_nr_regions.py +++ b/tools/testing/selftests/damon/damon_nr_regions.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def test_nr_regions(real_nr_regions, min_nr_regions, max_nr_regions): diff --git a/tools/testing/selftests/damon/damos_apply_interval.py b/tools/testing/selftests/damon/damos_apply_interval.py index 0f2f36584e48..0bf7768b2006 100755 --- a/tools/testing/selftests/damon/damos_apply_interval.py +++ b/tools/testing/selftests/damon/damos_apply_interval.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/damos_quota.py b/tools/testing/selftests/damon/damos_quota.py index 57c4937aaed2..879115a499bf 100755 --- a/tools/testing/selftests/damon/damos_quota.py +++ b/tools/testing/selftests/damon/damos_quota.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py index 661e4ba4765a..fed033a0afdf 100755 --- a/tools/testing/selftests/damon/damos_quota_goal.py +++ b/tools/testing/selftests/damon/damos_quota_goal.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/damos_tried_regions.py b/tools/testing/selftests/damon/damos_tried_regions.py index d6472e6a6e08..6941f87c10b1 100755 --- a/tools/testing/selftests/damon/damos_tried_regions.py +++ b/tools/testing/selftests/damon/damos_tried_regions.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py index 09552e91bc78..4622046fd011 100755 --- a/tools/testing/selftests/damon/drgn_dump_damon_status.py +++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py @@ -48,6 +48,37 @@ def attrs_to_dict(attrs): ['max_nr_regions', int], ]) +def filter_to_dict(damon_filter): + filter_type_keyword = { + 0: 'anon', + 1: 'memcg', + } + dict_ = { + 'type': filter_type_keyword[int(damon_filter.type)], + 'matching': bool(damon_filter.matching), + 'allow': bool(damon_filter.allow), + } + type_ = dict_['type'] + if type_ == 'memcg': + dict_['memcg_id'] = int(damon_filter.memcg_id) + return dict_ + +def filters_to_list(filters): + return [filter_to_dict(f) + for f in list_for_each_entry( + 'struct damon_filter', filters.address_of_(), 'list')] + +def probe_to_dict(probe): + return to_dict(probe, [ + ['weight', int], + ['filters', filters_to_list], + ]) + +def probes_to_list(probes): + return [probe_to_dict(p) + for p in list_for_each_entry( + 'struct damon_probe', probes.address_of_(), 'list')] + def addr_range_to_dict(addr_range): return to_dict(addr_range, [ ['start', int], @@ -199,6 +230,7 @@ def damon_ctx_to_dict(ctx): return to_dict(ctx, [ ['ops', ops_to_dict], ['attrs', attrs_to_dict], + ['probes', probes_to_list], ['adaptive_targets', targets_to_list], ['schemes', schemes_to_list], ['pause', bool], diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index 3ffa054b6386..66c826189320 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -178,6 +178,24 @@ def assert_monitoring_attrs_committed(attrs, dump): assert_true(dump['max_nr_regions'] == attrs.max_nr_regions, 'max_nr_regions', dump) +def assert_damon_filters_committed(filters, dump): + assert_true(len(dump) == len(filters.filters), 'probe filters', dump) + for idx, damon_filter in enumerate(filters.filters): + filter_dump = dump[idx] + assert_true(filter_dump['type'] == damon_filter.type_, 'type', + filter_dump) + assert_true(filter_dump['matching'] == damon_filter.matching, + 'matching', filter_dump) + assert_true(filter_dump['allow'] == damon_filter.allow, 'allow', + filter_dump) + +def assert_probes_committed(probes, dump): + assert_true(len(dump) == len(probes.probes), 'probes length', dump) + for idx, probe in enumerate(probes.probes): + probe_dump = dump[idx] + assert_true(probe.weight == probe_dump['weight'], 'weight', probe_dump) + assert_damon_filters_committed(probe.filters, probe_dump['filters']) + def assert_monitoring_target_committed(target, dump): # target.pid is the pid "number", while dump['pid'] is 'struct pid' # pointer, and hence cannot be compared. @@ -196,6 +214,7 @@ def assert_ctx_committed(ctx, dump): } assert_true(dump['ops']['id'] == ops_val[ctx.ops], 'ops_id', dump) assert_monitoring_attrs_committed(ctx.monitoring_attrs, dump['attrs']) + assert_probes_committed(ctx.monitoring_attrs.probes, dump['probes']) assert_monitoring_targets_committed(ctx.targets, dump['adaptive_targets']) assert_schemes_committed(ctx.schemes, dump['schemes']) assert_true(dump['pause'] == ctx.pause, 'pause', dump) @@ -300,7 +319,21 @@ def main(): intervals_goal=_damon_sysfs.IntervalsGoal( access_bp=400, aggrs=3, min_sample_us=5000, max_sample_us=10000000), - update_us=2000000), + update_us=2000000, + probes=_damon_sysfs.DamonProbes( + probes=[_damon_sysfs.DamonProbe( + weight=42, + filters=_damon_sysfs.DamonFilters( + filters=[ + _damon_sysfs.DamonFilter( + type_='anon', + matching=True, + allow=True, + ), + ]), + ), + ]), + ), schemes=[_damon_sysfs.Damos( action='pageout', access_pattern=_damon_sysfs.DamosAccessPattern( @@ -385,6 +418,10 @@ def main(): assert_ctxs_committed(kdamonds) kdamonds.stop() + for proc in (proc1, proc2, proc3): + proc.terminate() + proc.communicate() + test_memcg_filter_memcg_path_staging() if __name__ == '__main__': diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index f7fb94b84e71..b66593c9ac47 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -210,6 +210,7 @@ test_goal() ensure_write_succ "$fpath" "active_mem_bp" "valid input" ensure_write_succ "$fpath" "inactive_mem_bp" "valid input" ensure_write_succ "$fpath" "node_eligible_mem_bp" "valid input" + ensure_write_succ "$fpath" "hugepage_mem_bp" "valid input" ensure_write_fail "$fpath" "foo" "invalid input" ensure_file "$goal_dir/nid" "exist" "600" ensure_file "$goal_dir/path" "exist" "600" @@ -361,6 +362,32 @@ test_intervals() test_intervals_goal "$intervals_dir/intervals_goal" } +test_damon_prep() +{ + damon_prep_dir=$1 + ensure_file "$damon_prep_dir/prep_action" "exist" "600" + ensure_write_succ "$damon_prep_dir/prep_action" "set_pgidle" \ + "valid input" + ensure_write_fail "$damon_prep_dir/prep_action" "foo" "invalid input" +} + +test_damon_preps() +{ + preps_dir=$1 + ensure_dir "$preps_dir" "exist" + ensure_file "$preps_dir/nr_preps" "exist" "600" + ensure_write_succ "$preps_dir/nr_preps" "1" "valid input" + test_damon_prep "$preps_dir/0" + + ensure_write_succ "$preps_dir/nr_preps" "2" "valid input" + test_damon_prep "$preps_dir/0" + test_damon_prep "$preps_dir/1" + + ensure_write_succ "$preps_dir/nr_preps" "0" "valid input" + ensure_dir "$preps_dir/0" "not_exist" + ensure_dir "$preps_dir/1" "not_exist" +} + test_damon_filter() { damon_filter_dir=$1 @@ -392,6 +419,7 @@ test_probe() { probe_dir=$1 ensure_dir "$probe_dir" "exist" + test_damon_preps "$probe_dir/preps" test_damon_filters "$probe_dir/filters" } diff --git a/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh b/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh index 33a7ff43ed6c..34c37129c49f 100755 --- a/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh +++ b/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh @@ -41,5 +41,12 @@ if [ "$kmemleak_report" = "" ] then exit 0 fi +if ! echo "$kmemleak_report" | grep "memcg_path_store" --quiet +then + echo "[WARN] memleak found; apparently not from DAMON, though" + echo "$kmemleak_report" + exit 0 +fi + echo "$kmemleak_report" exit 1 diff --git a/tools/testing/selftests/damon/sysfs_no_op_commit_break.py b/tools/testing/selftests/damon/sysfs_no_op_commit_break.py index 2c65cffe6b54..81774ddc55ac 100755 --- a/tools/testing/selftests/damon/sysfs_no_op_commit_break.py +++ b/tools/testing/selftests/damon/sysfs_no_op_commit_break.py @@ -47,26 +47,27 @@ def main(): print('kdamond start failed: %s' % err) exit(1) - before_commit_status, err = \ - dump_damon_status_dict(kdamonds.kdamonds[0].pid) - if err is not None: - print('before-commit status dump failed: %s' % err) - exit(1) + try: + before_commit_status, err = \ + dump_damon_status_dict(kdamonds.kdamonds[0].pid) + if err is not None: + print('before-commit status dump failed: %s' % err) + exit(1) - kdamonds.kdamonds[0].commit() + kdamonds.kdamonds[0].commit() - after_commit_status, err = \ - dump_damon_status_dict(kdamonds.kdamonds[0].pid) - if err is not None: - print('after-commit status dump failed: %s' % err) - exit(1) + after_commit_status, err = \ + dump_damon_status_dict(kdamonds.kdamonds[0].pid) + if err is not None: + print('after-commit status dump failed: %s' % err) + exit(1) - if before_commit_status != after_commit_status: - print(f'before: {json.dumps(before_commit_status, indent=2)}') - print(f'after: {json.dumps(after_commit_status, indent=2)}') - exit(1) - - kdamonds.stop() + if before_commit_status != after_commit_status: + print(f'before: {json.dumps(before_commit_status, indent=2)}') + print(f'after: {json.dumps(after_commit_status, indent=2)}') + exit(1) + finally: + kdamonds.stop() if __name__ == '__main__': main() diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py index 28c887a0108f..625761c243b5 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py index 16fdc6e7fc56..36e7ae5f826d 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def pass_wss_estimation(sz_region): diff --git a/tools/testing/selftests/mm/.gitignore b/tools/testing/selftests/mm/.gitignore index fcd892ed21e3..a306d7754786 100644 --- a/tools/testing/selftests/mm/.gitignore +++ b/tools/testing/selftests/mm/.gitignore @@ -2,7 +2,9 @@ * !/**/ !*.c +*.mod.c !*.h +local_config.h !*.sh !.gitignore !Makefile diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 2d5366196e30..beacc0f87304 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -37,7 +37,8 @@ endif # LDLIBS. MAKEFLAGS += --no-builtin-rules -CFLAGS = -Wall -O2 -I $(top_srcdir) $(EXTRA_CFLAGS) $(KHDR_INCLUDES) $(TOOLS_INCLUDES) +CFLAGS = -Wall -O2 -I $(top_srcdir) -I $(top_srcdir)/tools/lib +CFLAGS += $(EXTRA_CFLAGS) $(KHDR_INCLUDES) $(TOOLS_INCLUDES) CFLAGS += -Wunreachable-code LDLIBS = -lrt -lpthread -lm @@ -57,8 +58,8 @@ endif TEST_GEN_FILES = cow TEST_GEN_FILES += compaction_test +TEST_GEN_FILES += gup TEST_GEN_FILES += gup_longterm -TEST_GEN_FILES += gup_test TEST_GEN_FILES += hmm-tests TEST_GEN_FILES += hugetlb-madvise TEST_GEN_FILES += hugetlb-mmap @@ -104,10 +105,10 @@ TEST_GEN_FILES += guard-regions TEST_GEN_FILES += merge TEST_GEN_FILES += rmap TEST_GEN_FILES += folio_split_race_test - -ifneq ($(ARCH),arm64) +TEST_GEN_FILES += folio_order_check +TEST_GEN_FILES += khugepaged_sync_check +TEST_GEN_FILES += khugepaged_race TEST_GEN_FILES += soft-dirty -endif ifeq ($(ARCH),x86_64) CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_32bit_program.c -m32) @@ -187,8 +188,10 @@ TEST_FILES += write_hugetlb_memory.sh include ../lib.mk -$(TEST_GEN_PROGS): vm_util.c hugepage_settings.c -$(TEST_GEN_FILES): vm_util.c hugepage_settings.c +$(TEST_GEN_PROGS): vm_util.c $(top_srcdir)/tools/lib/mm/hugepage_settings.c \ + $(top_srcdir)/tools/lib/mm/file_utils.c +$(TEST_GEN_FILES): vm_util.c $(top_srcdir)/tools/lib/mm/hugepage_settings.c \ + $(top_srcdir)/tools/lib/mm/file_utils.c $(OUTPUT)/uffd-stress: uffd-common.c $(OUTPUT)/uffd-unit-tests: uffd-common.c @@ -217,7 +220,7 @@ $(BINARIES_32): CFLAGS += -m32 -mxsave $(BINARIES_32): LDLIBS += -lrt -ldl -lm $(BINARIES_32): $(OUTPUT)/%_32: %.c $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ + $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $^ $(LDLIBS) -o $@ $(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-32,$(t)))) endif @@ -226,7 +229,7 @@ $(BINARIES_64): CFLAGS += -m64 -mxsave $(BINARIES_64): LDLIBS += -lrt -ldl $(BINARIES_64): $(OUTPUT)/%_64: %.c $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ + $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $^ $(LDLIBS) -o $@ $(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-64,$(t)))) endif diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 30d4ace7155a..b3f5377119cb 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -15,9 +15,9 @@ #include #include #include +#include #include "kselftest.h" -#include "hugepage_settings.h" #define MAP_SIZE_MB 100 #define MAP_SIZE (MAP_SIZE_MB * 1024 * 1024) diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 8aa5249d9bef..3264a828575b 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -29,7 +29,6 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" static size_t pagesize; static int pagemap_fd; diff --git a/tools/testing/selftests/mm/folio_order_check.c b/tools/testing/selftests/mm/folio_order_check.c new file mode 100644 index 000000000000..5eafbcc1b4f3 --- /dev/null +++ b/tools/testing/selftests/mm/folio_order_check.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Self-check for the vm_util folio-order helpers, is_backed_by_folio() and + * is_range_backed_by_order(), which the khugepaged mTHP cases use to detect + * collapse results. For every anon THP order the kernel supports, fault + * memory in with only that order enabled and require the helpers to report + * exactly that order. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "kselftest.h" +#include "vm_util.h" +#include + +static int pagemap_fd; +static int kpageflags_fd; + +static char *alloc_aligned(size_t size) +{ + size_t len = size * 2; + char *p, *aligned; + + p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (p == MAP_FAILED) + ksft_exit_fail_perror("mmap()"); + + aligned = (char *)ALIGN((uintptr_t)p, size); + if (aligned != p) + munmap(p, aligned - p); + if (aligned + size != p + len) + munmap(aligned + size, p + len - aligned - size); + + return aligned; +} + +static void check_order(int order) +{ + struct thp_settings settings = *thp_current_settings(); + size_t size = psize() << order; + bool ok = true; + char *p; + int i; + + for (i = 0; i < NR_ORDERS; i++) + settings.hugepages[i].enabled = THP_NEVER; + if (order) + settings.hugepages[order].enabled = THP_ALWAYS; + thp_push_settings(&settings); + + p = alloc_aligned(size); + *p = 1; + + if (!is_range_backed_by_order(p, size, order, pagemap_fd, kpageflags_fd)) { + ksft_print_msg("order %d not detected after fault\n", order); + ok = false; + } + + /* A lower order must be rejected: the folio is larger */ + if (order && is_range_backed_by_order(p, size, order - 1, + pagemap_fd, kpageflags_fd)) { + ksft_print_msg("order %d also reported as order %d\n", + order, order - 1); + ok = false; + } + + /* A large folio must not pass as order 0 */ + if (order && is_range_backed_by_order(p, size, 0, + pagemap_fd, kpageflags_fd)) { + ksft_print_msg("order %d also reported as order 0\n", order); + ok = false; + } + + munmap(p, size); + thp_pop_settings(); + + ksft_test_result(ok, "order %d classified\n", order); +} + +int main(void) +{ + struct thp_settings settings; + unsigned long orders; + int order; + + ksft_print_header(); + + if (!thp_available()) + ksft_exit_skip("Transparent Hugepages not available\n"); + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_perror("open(/proc/self/pagemap)"); + kpageflags_fd = open("/proc/kpageflags", O_RDONLY); + if (kpageflags_fd < 0) + ksft_exit_skip("open(/proc/kpageflags) requires root\n"); + + orders = thp_supported_orders(); + if (!orders) + ksft_exit_skip("No supported THP orders\n"); + + ksft_set_plan(__builtin_popcountl(orders) + 1); + + thp_save_settings(); + thp_read_settings(&settings); + /* Base of the settings stack; the bottom entry is never popped */ + thp_push_settings(&settings); + + check_order(0); + for (order = 1; order < NR_ORDERS; order++) { + if (!(orders & (1UL << order))) + continue; + check_order(order); + } + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/folio_split_race_test.c b/tools/testing/selftests/mm/folio_split_race_test.c index 45b84f7b364e..e4660bf89b62 100644 --- a/tools/testing/selftests/mm/folio_split_race_test.c +++ b/tools/testing/selftests/mm/folio_split_race_test.c @@ -25,7 +25,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" uint64_t page_size; uint64_t pmd_pagesize; @@ -269,6 +268,4 @@ int main(void) NUM_ITERATIONS); ksft_exit(iter == NUM_ITERATIONS); - - return 0; } diff --git a/tools/testing/selftests/mm/guard-regions.c b/tools/testing/selftests/mm/guard-regions.c index 5c8ec3ca75d7..f7d53ea3c252 100644 --- a/tools/testing/selftests/mm/guard-regions.c +++ b/tools/testing/selftests/mm/guard-regions.c @@ -21,7 +21,6 @@ #include #include #include "vm_util.h" -#include "hugepage_settings.h" #include "../pidfd/pidfd.h" @@ -2257,8 +2256,18 @@ TEST_F(guard_regions, smaps) char *ptr, *ptr2; int i; - /* Map a region. */ - ptr = mmap_(self, variant, NULL, 10 * page_size, PROT_READ | PROT_WRITE, 0, 0); + /* Map then unmap placeholder to avoid adjacent merges */ + ptr = mmap_(self, variant, NULL, 12 * page_size, PROT_NONE, 0, 0); + ASSERT_NE(ptr, MAP_FAILED); + ASSERT_EQ(munmap(ptr, 12 * page_size), 0); + + /* + * Map a region for the test. Since the preceding temporary mapping + * succeeded, this mapping should also succeed without merging with + * adjacent VMAs. + */ + ptr = mmap_(self, variant, ptr + page_size, 10 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED, 0); ASSERT_NE(ptr, MAP_FAILED); /* We shouldn't yet see a guard flag. */ diff --git a/tools/testing/selftests/mm/gup.c b/tools/testing/selftests/mm/gup.c new file mode 100644 index 000000000000..a6a8ca47d12e --- /dev/null +++ b/tools/testing/selftests/mm/gup.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +#define __SANE_USERSPACE_TYPES__ // Use ll64 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "vm_util.h" +#include "kselftest_harness.h" + +#define MB (1UL << 20) + +/* Just the flags we need, copied from the kernel internals. */ +#define FOLL_WRITE 0x01 /* check pte is writable */ + +/* Page counts exercising single, THP-batch, partial, and full-mapping GUP. */ +static const int nr_pages_list[] = { 1, 512, 123, -1 }; + +#define GUP_TEST_FILE "/sys/kernel/debug/gup_test" +#define NR_HUGETLB_PAGES 2 + +static unsigned long hp_size; + +FIXTURE(gup_test) +{ + int gup_fd; + char *addr; + unsigned long size; +}; + +FIXTURE_VARIANT(gup_test) +{ + bool thp; + bool hugetlb; + bool write; + bool shared; +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write) +{ + .thp = false, + .hugetlb = false, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read) +{ + .thp = false, + .hugetlb = false, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write_thp) +{ + .thp = true, + .hugetlb = false, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read_thp) +{ + .thp = true, + .hugetlb = false, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write) +{ + .thp = false, + .hugetlb = false, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read) +{ + .thp = false, + .hugetlb = false, + .write = false, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write_thp) +{ + .thp = true, + .hugetlb = false, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read_thp) +{ + .thp = true, + .hugetlb = false, + .write = false, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = false, + .shared = true, +}; + +FIXTURE_SETUP(gup_test) +{ + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS; + char *p; + + self->size = 128 * MB; + + if (variant->hugetlb) { + if (!hp_size) + SKIP(return, "HugeTLB not available\n"); + + if (hugetlb_free_default_pages() < NR_HUGETLB_PAGES) + SKIP(return, "Not enough huge pages\n"); + + self->size = NR_HUGETLB_PAGES * hp_size; + mmap_flags |= MAP_HUGETLB; + } + + if (variant->shared) + mmap_flags = (mmap_flags & ~MAP_PRIVATE) | MAP_SHARED; + + /* gup_fd has to be >= 0. Already checked in main() */ + self->gup_fd = open(GUP_TEST_FILE, O_RDWR); + ASSERT_GE(self->gup_fd, 0); + + self->addr = mmap(NULL, self->size, PROT_READ | PROT_WRITE, + mmap_flags, -1, 0); + + ASSERT_NE(self->addr, MAP_FAILED) { + int err = errno; + + close(self->gup_fd); + TH_LOG("mmap failed: %s", strerror(err)); + } + + if (variant->thp) + madvise(self->addr, self->size, MADV_HUGEPAGE); + else if (!variant->hugetlb) + madvise(self->addr, self->size, MADV_NOHUGEPAGE); + + for (p = self->addr; (unsigned long)p < (unsigned long)self->addr + + self->size; p += psize()) + p[0] = 0; +} + +FIXTURE_TEARDOWN(gup_test) +{ + munmap(self->addr, self->size); + close(self->gup_fd); +} + +static void run_gup_cmd(struct __test_metadata *_metadata, + FIXTURE_DATA(gup_test) *self, + const FIXTURE_VARIANT(gup_test) *variant, + unsigned long command) +{ + int i; + + for (i = 0; i < (int)ARRAY_SIZE(nr_pages_list); i++) { + struct gup_test gup = { + .addr = (unsigned long)self->addr, + .size = self->size, + .nr_pages_per_call = nr_pages_list[i] < 0 ? + self->size / psize() : nr_pages_list[i], + .gup_flags = variant->write ? FOLL_WRITE : 0, + }; + + TH_LOG("nr_pages_per_call=%u", gup.nr_pages_per_call); + ASSERT_EQ(ioctl(self->gup_fd, command, &gup), 0); + ASSERT_EQ(gup.size, self->size); + } +} + +TEST_F(gup_test, get_user_pages) +{ + run_gup_cmd(_metadata, self, variant, GUP_BASIC_TEST); +} + +TEST_F(gup_test, pin_user_pages) +{ + run_gup_cmd(_metadata, self, variant, PIN_BASIC_TEST); +} + +TEST_F(gup_test, get_user_pages_fast) +{ + run_gup_cmd(_metadata, self, variant, GUP_FAST_BENCHMARK); +} + +TEST_F(gup_test, pin_user_pages_fast) +{ + run_gup_cmd(_metadata, self, variant, PIN_FAST_BENCHMARK); +} + +TEST_F(gup_test, pin_user_pages_longterm) +{ + run_gup_cmd(_metadata, self, variant, PIN_LONGTERM_BENCHMARK); +} + +int main(int argc, char **argv) +{ + const int fd = open(GUP_TEST_FILE, O_RDWR); + + if (fd == -1) { + ksft_print_header(); + if (errno == EACCES) + ksft_exit_skip("Please run this test as root\n"); + if (errno == ENOENT) { + DIR *debugfs = opendir("/sys/kernel/debug"); + + if (!debugfs) + ksft_exit_skip("Mount debugfs at /sys/kernel/debug\n"); + closedir(debugfs); + ksft_exit_skip("Check CONFIG_GUP_TEST in kernel config\n"); + } + ksft_exit_fail_msg("Failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno)); + } + close(fd); + + hp_size = default_huge_page_size(); + if (hp_size) + hugetlb_setup_default(NR_HUGETLB_PAGES); + + return test_harness_run(argc, argv); +} diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index 510de93be681..c9d8b4491263 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -29,7 +29,6 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" static size_t pagesize; static int nr_hugetlbsizes; diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index e2642eca0d02..e5f273ca84c1 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -10,9 +10,6 @@ * bugs. */ -#include "kselftest_harness.h" -#include "hugepage_settings.h" - #include #include #include @@ -33,6 +30,9 @@ #include #include #include +#include + +#include "kselftest_harness.h" /* * This is a private UAPI to the kernel test module so it isn't exported @@ -65,7 +65,6 @@ enum { #define HMM_PATH_MAX 64 #define NTIMES 10 -#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) /* Just the flags we need, copied from mm.h: */ #ifndef FOLL_WRITE diff --git a/tools/testing/selftests/mm/hugetlb-madvise.c b/tools/testing/selftests/mm/hugetlb-madvise.c index 555b4b3d1430..57cf790ca478 100644 --- a/tools/testing/selftests/mm/hugetlb-madvise.c +++ b/tools/testing/selftests/mm/hugetlb-madvise.c @@ -14,7 +14,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define MIN_FREE_PAGES 20 #define NR_HUGE_PAGES 10 /* common number of pages to map/allocate */ diff --git a/tools/testing/selftests/mm/hugetlb-mmap.c b/tools/testing/selftests/mm/hugetlb-mmap.c index 2edbc992e6bc..8853271fc825 100644 --- a/tools/testing/selftests/mm/hugetlb-mmap.c +++ b/tools/testing/selftests/mm/hugetlb-mmap.c @@ -18,7 +18,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define LENGTH (256UL*1024*1024) #define PROTECTION (PROT_READ | PROT_WRITE) diff --git a/tools/testing/selftests/mm/hugetlb-mremap.c b/tools/testing/selftests/mm/hugetlb-mremap.c index ed3d92e862d8..9b724af66e93 100644 --- a/tools/testing/selftests/mm/hugetlb-mremap.c +++ b/tools/testing/selftests/mm/hugetlb-mremap.c @@ -26,7 +26,6 @@ #include #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #define DEFAULT_LENGTH_MB 10UL #define MB_TO_BYTES(x) (x * 1024 * 1024) diff --git a/tools/testing/selftests/mm/hugetlb-shm.c b/tools/testing/selftests/mm/hugetlb-shm.c index 3ff7f062b7eb..f4514da49e1d 100644 --- a/tools/testing/selftests/mm/hugetlb-shm.c +++ b/tools/testing/selftests/mm/hugetlb-shm.c @@ -29,7 +29,6 @@ #include #include "vm_util.h" -#include "hugepage_settings.h" #define LENGTH (256UL*1024*1024) diff --git a/tools/testing/selftests/mm/hugetlb-soft-offline.c b/tools/testing/selftests/mm/hugetlb-soft-offline.c index bc202e4ed2bd..d9565219378a 100644 --- a/tools/testing/selftests/mm/hugetlb-soft-offline.c +++ b/tools/testing/selftests/mm/hugetlb-soft-offline.c @@ -11,6 +11,7 @@ #define _GNU_SOURCE #include +#include #include #include #include @@ -21,9 +22,10 @@ #include #include #include +#include #include "kselftest.h" -#include "hugepage_settings.h" +#include "vm_util.h" #ifndef MADV_SOFT_OFFLINE #define MADV_SOFT_OFFLINE 101 @@ -31,6 +33,8 @@ #define EPREFIX " !!! " +#define ENABLE_SOFT_OFFLINE_PATH "/proc/sys/vm/enable_soft_offline" + static int do_soft_offline(int fd, size_t len, int expect_errno) { char *filemap = NULL; @@ -77,26 +81,28 @@ untruncate: return ret; } -static int set_enable_soft_offline(int value) +static unsigned long orig_enable_soft_offline = -1UL; + +/* + * Runs from an atexit handler, so it must not call anything that + * exits on failure. + */ +static void restore_enable_soft_offline(void) { - char cmd[256] = {0}; - FILE *cmdfile = NULL; + char buf[24]; + int fd, len; - if (value != 0 && value != 1) - return -EINVAL; + if (orig_enable_soft_offline == -1UL) + return; - sprintf(cmd, "echo %d > /proc/sys/vm/enable_soft_offline", value); - cmdfile = popen(cmd, "r"); - - if (cmdfile) - ksft_print_msg("enable_soft_offline => %d\n", value); - else { - ksft_perror(EPREFIX "failed to set enable_soft_offline"); - return errno; - } - - pclose(cmdfile); - return 0; + len = snprintf(buf, sizeof(buf), "%lu", orig_enable_soft_offline); + fd = open(ENABLE_SOFT_OFFLINE_PATH, O_WRONLY); + if (fd < 0) + return; + if (write(fd, buf, len) != len) + ksft_print_msg("failed to restore enable_soft_offline: %s\n", + strerror(errno)); + close(fd); } static int create_hugetlbfs_file(struct statfs *file_stat) @@ -145,10 +151,10 @@ static void test_soft_offline_common(int enable_soft_offline) hugepagesize_kb = file_stat.f_bsize / 1024; ksft_print_msg("Hugepagesize is %ldkB\n", hugepagesize_kb); - if (set_enable_soft_offline(enable_soft_offline) != 0) { - close(fd); - ksft_exit_fail_msg("Failed to set enable_soft_offline\n"); - } + ret = write_num(ENABLE_SOFT_OFFLINE_PATH, enable_soft_offline); + if (ret) + ksft_exit_fail_msg("Failed to write to %s: %s\n", + ENABLE_SOFT_OFFLINE_PATH, strerror(-ret)); nr_hugepages_before = hugetlb_nr_default_pages(); @@ -185,6 +191,8 @@ static void test_soft_offline_common(int enable_soft_offline) int main(int argc, char **argv) { + int ret; + ksft_print_header(); if (!hugetlb_setup_default(8)) @@ -192,6 +200,13 @@ int main(int argc, char **argv) ksft_set_plan(2); + ret = read_num(ENABLE_SOFT_OFFLINE_PATH, &orig_enable_soft_offline); + if (ret) + ksft_exit_fail_msg("Failed to read %s: %s\n", + ENABLE_SOFT_OFFLINE_PATH, strerror(-ret)); + + atexit(restore_enable_soft_offline); + test_soft_offline_common(1); test_soft_offline_common(0); diff --git a/tools/testing/selftests/mm/hugetlb_dio.c b/tools/testing/selftests/mm/hugetlb_dio.c index fb4600570e13..9495974eccbe 100644 --- a/tools/testing/selftests/mm/hugetlb_dio.c +++ b/tools/testing/selftests/mm/hugetlb_dio.c @@ -20,7 +20,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #ifndef STATX_DIOALIGN #define STATX_DIOALIGN 0x00002000U diff --git a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c index 2dc158054f66..56c5a8533e9d 100644 --- a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c +++ b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c @@ -10,7 +10,6 @@ #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define INLOOP_ITER 100 diff --git a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c index f94549efcc6f..1d111f42dd59 100644 --- a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c +++ b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c @@ -3,18 +3,6 @@ * A test case that must run on a system with one and only one huge page available. * # echo 1 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages * - * During setup, the test allocates the only available page, and starts three threads: - * - thread1: - * * madvise(MADV_DONTNEED) on the allocated huge page - * - thread 2: - * * Write to the allocated huge page - * - thread 3: - * * Try to allocated an extra huge page (which must not available) - * - * The test fails if thread3 is able to allocate a page. - * - * Touching the first page after thread3's allocation will raise a SIGBUS - * * Author: Breno Leitao */ #include @@ -22,10 +10,10 @@ #include #include #include +#include #include #include "vm_util.h" -#include "hugepage_settings.h" #define INLOOP_ITER 100 @@ -74,7 +62,21 @@ void *map_extra(void *unused) return NULL; } -int main(void) +/* During setup in main, the only available page was allocated. This test then + * starts three threads: + * + * - thread1: + * * madvise(MADV_DONTNEED) on the allocated huge page + * - thread 2: + * * Write to the allocated huge page + * - thread 3: + * * Try to allocated an extra huge page (which must not available) + * + * The test fails if thread3 is able to allocate a page. + * + * Touching the first page after thread3's allocation will raise a SIGBUS + */ +void test_madv_vs_map(void) { pthread_t thread1, thread2, thread3; void *ret; @@ -85,13 +87,6 @@ int main(void) */ int max = 10; - ksft_print_header(); - ksft_set_plan(1); - - if (!hugetlb_setup_default_exact(1)) - ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", - hugetlb_free_default_pages()); - mmap_size = default_huge_page_size(); while (max--) { @@ -100,7 +95,7 @@ int main(void) -1, 0); if ((unsigned long)huge_ptr == -1) - ksft_exit_fail_msg("Failed to allocate huge page\n"); + ksft_exit_fail_perror("Failed to allocate huge page"); pthread_create(&thread1, NULL, madv, NULL); pthread_create(&thread2, NULL, touch, NULL); @@ -120,5 +115,108 @@ int main(void) } ksft_test_result_pass("No unexpected huge page allocations\n"); +} + +/* We create a child process, then unmap the page in the parent while the child + * waits and verify that there is no underflow of the reserved count. We also + * verify that after the child exits, the reserved count is properly restored. + */ +void test_underflow(void) +{ + pid_t pid; + int pipe_fds[2]; + unsigned long nr_reserved = 0; + + huge_ptr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + + if ((unsigned long)huge_ptr == -1) + ksft_exit_fail_perror("Failed to allocate huge page"); + + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved != 1) + ksft_exit_fail_msg("Unexpected number of reserved pages: %lu, expected 1\n", + nr_reserved); + + /* Force the fault to ensure the reservation is consumed */ + *huge_ptr = 0; + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved != 0) + ksft_exit_fail_msg("Unexpected number of reserved pages: %lu, expected 0\n", + nr_reserved); + + if (pipe(pipe_fds) != 0) + ksft_exit_fail_perror("pipe failed"); + + pid = fork(); + if (pid < 0) + ksft_exit_fail_perror("fork failed"); + + if (pid == 0) { + /* Child: Simply wait for the parent */ + char b; + + close(pipe_fds[1]); + if (read(pipe_fds[0], &b, 1) < 0) + ksft_perror("child read failed"); + /* Let the parent do the cleanup */ + _exit(0); + } + + /* Parent */ + close(pipe_fds[0]); + + /* First unmap, this will close the vma */ + if (munmap(huge_ptr, mmap_size) != 0) { + ksft_perror("munmap failed"); + goto err_cleanup; + } + + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved == 0) { + ksft_test_result_pass("Underflow not present!\n"); + } else { + ksft_test_result_fail("Unexpected HugePages_Rsvd=%ld after munmap, should be 0\n", + nr_reserved); + goto err_cleanup; + } + /* Make the child exit, this should restore HugePages_Rsvd to 0 */ + if (write(pipe_fds[1], &nr_reserved, 1) < 0) { + /* If write failed, the child is likely already gone */ + ksft_exit_fail_perror("write failed"); + } + close(pipe_fds[1]); + if (waitpid(pid, NULL, 0) <= 0) + ksft_exit_fail_msg("waitpid failed\n"); + + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved == 0) + ksft_test_result_pass("After the child dies, HugePages_Rsvd is properly set to 0\n"); + else + ksft_exit_fail_msg("Unexpected HugePages_Rsvd=%ld after the child termination munmap, should be 0\n", nr_reserved); + + return; + +err_cleanup: + if (write(pipe_fds[1], &nr_reserved, 1) < 0) + ksft_exit_fail_perror("write failed"); + if (waitpid(pid, NULL, 0) <= 0) + ksft_exit_fail_perror("waitpid failed"); + + ksft_exit_fail(); +} + +int main(void) +{ + ksft_print_header(); + ksft_set_plan(3); + + if (!hugetlb_setup_default_exact(1)) + ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", + hugetlb_free_default_pages()); + + test_madv_vs_map(); + test_underflow(); + ksft_finished(); } diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 1d2d6bd72fd2..a2ac3b3ca5de 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -22,7 +22,6 @@ #include "linux/magic.h" #include "vm_util.h" -#include "hugepage_settings.h" #define BASE_ADDR ((void *)(1UL << 30)) static unsigned long hpage_pmd_size; @@ -31,6 +30,11 @@ static unsigned long page_size; static int hpage_pmd_nr; static int anon_order; static int collapse_order; +static bool collapse_order_set; +static int collapse_orders[NR_ORDERS]; +static int nr_collapse_orders; +static int pagemap_fd = -1; +static int kpageflags_fd = -1; #define PID_SMAPS "/proc/self/smaps" #define TEST_FILE "collapse_test_file" @@ -122,6 +126,7 @@ static void get_finfo(const char *dir) char buf[1 << 10]; char path[PATH_MAX]; char *str, *end; + int ret; finfo.dir = dir; if (stat(finfo.dir, &path_stat)) @@ -142,8 +147,9 @@ static void get_finfo(const char *dir) major(path_stat.st_dev), minor(path_stat.st_dev)) >= sizeof(path)) ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); - if (!read_file(path, buf, sizeof(buf))) - ksft_exit_fail_perror("read_file(uevent)"); + ret = read_file(path, buf, sizeof(buf)); + if (ret) + ksft_exit_fail_msg("read_file(%s): %s\n", path, strerror(-ret)); if (strstr(buf, "DEVTYPE=disk")) { /* Found it */ if (snprintf(finfo.dev_queue_read_ahead_path, @@ -220,6 +226,29 @@ err_out: return swap; } +static bool swapout_range(void *p, unsigned long size) +{ + int i; + + /* keep khugepaged from collapsing the range and swapping it back in */ + if (madvise(p, size, MADV_NOHUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_NOHUGEPAGE)"); + + /* + * Retry several times because MADV_PAGEOUT is best effort. Sleep + * between the retries to give outstanding writeback a chance to + * finish. + */ + for (i = 0; i < 40; i++) { + if (madvise(p, size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); + if (check_swap(p, size)) + return true; + usleep(50 * 1000); + } + return false; +} + static void *alloc_mapping(int nr) { void *p; @@ -324,7 +353,7 @@ static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) { const int open_opt = setup == FILE_SETUP_READ_ONLY_FS ? O_RDONLY : O_RDWR; const int mmap_prot = setup == FILE_SETUP_READ_ONLY_FS ? PROT_READ : (PROT_READ | PROT_WRITE); - int fd; + int fd, ret; void *p; unsigned long size; @@ -337,21 +366,15 @@ static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) ksft_exit_fail_perror("open()"); size = nr_hpages * hpage_pmd_size; - if (ftruncate(fd, size)) { - perror("ftruncate()"); - exit(EXIT_FAILURE); - } + if (ftruncate(fd, size)) + ksft_exit_fail_perror("ftruncate()"); p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (p != BASE_ADDR) { - perror("mmap()"); - exit(EXIT_FAILURE); - } + if (p != BASE_ADDR) + ksft_exit_fail_perror("mmap()"); fill_memory(p, 0, size); - if (msync(p, size, MS_SYNC)) { - perror("msync()"); - exit(EXIT_FAILURE); - } + if (msync(p, size, MS_SYNC)) + ksft_exit_fail_perror("msync()"); close(fd); munmap(p, size); success("OK"); @@ -368,7 +391,11 @@ static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) ksft_exit_fail_perror("mmap()"); /* Drop page cache */ - write_file("/proc/sys/vm/drop_caches", "3", 2); + ret = write_file("/proc/sys/vm/drop_caches", "3", 2); + if (ret) + ksft_exit_fail_msg("write_file(drop_caches): %s\n", + strerror(-ret)); + success("OK"); return p; } @@ -426,7 +453,7 @@ static bool file_check_huge(void *addr, size_t len, int nr_hpages, case VMA_SHMEM: return check_huge_shmem(addr, len, nr_hpages, hpage_size); default: - exit(EXIT_FAILURE); + ksft_exit_fail_msg("Unknown VMA type\n"); return false; } } @@ -516,8 +543,8 @@ static bool is_anon(struct mem_ops *ops) static void __madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { - int ret; struct thp_settings settings = *thp_current_settings(); + int ret, i; ksft_print_msg("%s...", msg); @@ -533,6 +560,10 @@ static void __madvise_collapse(const char *msg, char *p, int nr_hpages, */ settings.thp_enabled = THP_NEVER; settings.shmem_enabled = SHMEM_NEVER; + for (i = 0; i < NR_ORDERS; i++) { + settings.hugepages[i].enabled = THP_NEVER; + settings.shmem_hugepages[i].enabled = SHMEM_NEVER; + } thp_push_settings(&settings); /* Clear VM_NOHUGEPAGE */ @@ -562,8 +593,11 @@ static bool wait_for_scan(const char *msg, char *p, size_t len, int nr_hpages, int collap_order, struct mem_ops *ops) { unsigned long hpage_size = page_size << collap_order; - int full_scans; - int timeout = 6; /* 3 seconds */ + unsigned long bytes = (unsigned long)nr_hpages * hpage_size; + int timeout, full_scans; + + /* Half-second ticks: three seconds floor, plus a second per 128M */ + timeout = 6 + 2 * (bytes / (128UL << 20)); /* Sanity check */ if (!ops->check_huge(p, len, 0, hpage_size)) @@ -826,12 +860,10 @@ static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_op ops->fault(p, 0, hpage_pmd_size); ksft_print_msg("Swapout one page..."); - if (madvise(p, page_size, MADV_PAGEOUT)) - ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); - if (check_swap(p, page_size)) { + if (swapout_range(p, page_size)) { success("OK"); } else { - fail("Fail"); + skip("Could not swap out"); goto out; } @@ -852,12 +884,10 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o ops->fault(p, 0, hpage_pmd_size); ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); - if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) - ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); - if (check_swap(p, (max_ptes_swap + 1) * page_size)) { + if (swapout_range(p, (max_ptes_swap + 1) * page_size)) { success("OK"); } else { - fail("Fail"); + skip("Could not swap out"); goto out; } @@ -869,12 +899,10 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o ops->fault(p, 0, hpage_pmd_size); ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); - if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) - ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); - if (check_swap(p, max_ptes_swap * page_size)) { + if (swapout_range(p, max_ptes_swap * page_size)) { success("OK"); } else { - fail("Fail"); + skip("Could not swap out"); goto out; } @@ -941,6 +969,16 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops void *p; int i; + /* + * This needs hpage_pmd_nr PMD-order allocations in a row, which the + * allocator will not supply if the PMD is very large. + */ + if (hpage_pmd_size > (32UL << 20)) { + ksft_test_result_skip("%s: PMD too large for fault-time THP construction\n", + __func__); + return; + } + p = ops->setup_area(1); ksft_print_msg("Construct PTE page table full of different PTE-mapped compound pages\n"); for (i = 0; i < hpage_pmd_nr; i++) { @@ -1131,6 +1169,109 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops ksft_test_result_report(exit_status, "%s\n", __func__); } +/* + * The parent writes to the fork-shared range throughout the child's + * collapse. CoW must keep the two apart: the child sees the pre-fork + * content, the parent only its own writes. + */ +static void collapse_fork_cow_race(struct collapse_context *c, struct mem_ops *ops) +{ + const int stride = page_size / sizeof(int); + int wstatus, child_status, i, n; + unsigned long shared; + volatile int *ip; + pid_t child; + int sync[2]; + char go = 1; + void *p; + + /* At a page per 10 ms, 64 pages spread the writes across the collapse */ + n = 64; + shared = n * page_size; + + p = ops->setup_area(1); + /* Shared prefix, with the pre-fork pattern */ + ops->fault(p, 0, shared); + if (pipe(sync)) + ksft_exit_fail_perror("pipe()"); + + /* A volatile pointer so the stores are not merged or dropped */ + ip = p; + + ksft_print_msg("Fork, collapse in the child while the parent rewrites..."); + child = fork(); + if (!child) { + int collapse_status; + + close(sync[0]); + /* Private remainder */ + ops->fault(p, shared, hpage_pmd_size); + /* Start the parent unsharing, and give it a head start */ + if (write(sync[1], &go, 1) != 1) + _exit(KSFT_FAIL); + usleep(5000); + c->collapse("Collapse a range the parent is writing to", + p, 1, ops, true); + collapse_status = exit_status; + for (i = 0; i < n; i++) + if (ip[i * stride] != i + 0xdead0000) + break; + if (i == n) + success("OK"); + else + fail("Fail: child content"); + /* The content check must not bury a failed collapse */ + if (exit_status != KSFT_FAIL) + exit_status = collapse_status; + ops->cleanup_area(p, hpage_pmd_size); + _exit(exit_status); + } + + close(sync[1]); + if (read(sync[0], &go, 1) != 1) + ksft_exit_fail_msg("child never reached the collapse\n"); + close(sync[0]); + + /* + * Unshare one page at a time: a burst would break CoW on the whole + * range before the collapse starts, leaving nothing shared to collapse. + */ + i = 0; + for (;;) { + pid_t ret; + + if (i < n) + ip[i * stride] = i + 0xbeef0000; + i++; + usleep(10 * 1000); + ret = waitpid(child, &wstatus, WNOHANG); + if (ret == child) + break; + if (ret < 0) + ksft_exit_fail_perror("waitpid()"); + } + + /* Finish whatever the paced sweep did not reach */ + for (; i < n; i++) + ip[i * stride] = i + 0xbeef0000; + /* A child that died reading the racing pages is a failure, not a zero */ + child_status = WIFEXITED(wstatus) ? WEXITSTATUS(wstatus) : KSFT_FAIL; + + ksft_print_msg("Check the parent sees only its own writes..."); + for (i = 0; i < n; i++) + if (ip[i * stride] != i + 0xbeef0000) + break; + if (i == n) + success("OK"); + else + fail("Fail: parent content"); + ops->cleanup_area(p, hpage_pmd_size); + /* The parent's check must not bury the child's verdict */ + if (exit_status != KSFT_FAIL) + exit_status = child_status; + ksft_test_result_report(exit_status, "%s\n", __func__); +} + static void madvise_collapse_existing_thps(struct collapse_context *c, struct mem_ops *ops) { @@ -1176,6 +1317,200 @@ static void madvise_retracted_page_tables(struct collapse_context *c, ksft_test_result_report(exit_status, "%s\n", __func__); } +/* Smallest order khugepaged will consider for mTHP collapse */ +#define MIN_MTHP_ORDER 2 + +/* Time budget for one khugepaged pass in the collapse_order_* cases */ +#define MTHP_PASS_TIMEOUT_S 30 + +static size_t mthp_window_size(void) +{ + return page_size << collapse_order; +} + +static void mthp_push_target_order(void) +{ + struct thp_settings settings = *thp_current_settings(); + int i; + + /* + * Only the target order, and only for madvise: the cases fault their + * region first, so the sources stay order 0 whatever -s asked for. + */ + settings.thp_enabled = THP_NEVER; + for (i = 0; i < NR_ORDERS; i++) + settings.hugepages[i].enabled = THP_NEVER; + settings.hugepages[collapse_order].enabled = THP_MADVISE; + thp_push_settings(&settings); +} + +static bool all_windows_at_order(void *p, size_t len) +{ + return is_range_backed_by_order(p, len, collapse_order, + pagemap_fd, kpageflags_fd); +} + +static bool any_window_at_order(void *p, size_t len) +{ + size_t window = mthp_window_size(); + char *addr = p; + + for (; len >= window; addr += window, len -= window) { + if (all_windows_at_order(addr, window)) + return true; + } + return false; +} + +static void collapse_order_single_window(struct collapse_context *c, + struct mem_ops *ops) +{ + size_t window = mthp_window_size(); + void *p; + + mthp_push_target_order(); + + p = ops->setup_area(1); + ops->fault(p, window, 2 * window); + if (any_window_at_order(p, hpage_pmd_size)) + ksft_exit_fail_msg("Unexpected large folio after fault\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse one fully populated window..."); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p + window, window) && + !any_window_at_order(p, window) && + !any_window_at_order(p + 2 * window, + hpage_pmd_size - 2 * window)) + success("OK"); + else + fail("Fail"); + + validate_memory(p, window, 2 * window); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + +static void collapse_order_partial_window(struct collapse_context *c, + struct mem_ops *ops) +{ + void *p; + + mthp_push_target_order(); + + p = ops->setup_area(1); + ops->fault(p, 0, page_size); + if (any_window_at_order(p, hpage_pmd_size)) + ksft_exit_fail_msg("Unexpected large folio after fault\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse window with single PTE entry present..."); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p, mthp_window_size())) + success("OK"); + else + fail("Fail"); + + validate_memory(p, 0, page_size); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + +static void collapse_order_max_ptes_none(struct collapse_context *c, + struct mem_ops *ops) +{ + struct thp_settings settings; + size_t window = mthp_window_size(); + void *p; + + mthp_push_target_order(); + settings = *thp_current_settings(); + settings.khugepaged.max_ptes_none = 0; + thp_push_settings(&settings); + + p = ops->setup_area(1); + ops->fault(p, 0, 2 * window - page_size); + if (any_window_at_order(p, hpage_pmd_size)) + ksft_exit_fail_msg("Unexpected large folio after fault\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse full window, not the one missing a page..."); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p, window) && + !any_window_at_order(p + window, window)) + success("OK"); + else + fail("Fail"); + + validate_memory(p, 0, 2 * window - page_size); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + +static void collapse_order_mixed_sources(struct collapse_context *c, + struct mem_ops *ops) +{ + int source_order = anon_order ? anon_order : MIN_MTHP_ORDER; + struct thp_settings settings; + void *p; + + if (source_order >= collapse_order || + !(thp_supported_orders() & (1UL << source_order))) { + ksft_test_result_skip("%s: no supported source order below target\n", + __func__); + return; + } + + mthp_push_target_order(); + + settings = *thp_current_settings(); + settings.hugepages[source_order].enabled = THP_ALWAYS; + thp_push_settings(&settings); + p = ops->setup_area(1); + ops->fault(p, 0, hpage_pmd_size); + thp_pop_settings(); + + /* + * The allocator can fall back to smaller folios under fragmentation; + * having nothing to collapse from is not a failure. + */ + if (!is_range_backed_by_order(p, hpage_pmd_size, source_order, + pagemap_fd, kpageflags_fd)) { + ksft_print_msg("No order-%d sources to collapse...", source_order); + skip("Skip"); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); + return; + } + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse region backed by order-%d sources...", + source_order); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p, hpage_pmd_size)) + success("OK"); + else + fail("Fail"); + + validate_memory(p, 0, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + static void usage(void) { fprintf(stderr, "\nUsage: ./khugepaged [OPTIONS] [dir]\n\n"); @@ -1193,11 +1528,14 @@ static void usage(void) fprintf(stderr, "\t\t-s: mTHP size, expressed as page order.\n"); fprintf(stderr, "\t\t Defaults to 0. Use this size for anon or shmem allocations.\n"); fprintf(stderr, "\t\t-c: collapse order for mTHP collapse, expressed as page order.\n"); + fprintf(stderr, "\t\t Defaults to every supported order below the PMD.\n"); + fprintf(stderr, "\t\t -s, if set, is the source order for the mixed-source case.\n"); exit(1); } static void parse_test_type(int argc, char **argv) { + bool mthp_context_implied = false; int opt; char *buf; const char *token; @@ -1209,6 +1547,7 @@ static void parse_test_type(int argc, char **argv) break; case 'c': collapse_order = atoi(optarg); + collapse_order_set = true; break; case 'h': default: @@ -1216,29 +1555,43 @@ static void parse_test_type(int argc, char **argv) } } + /* + * Both orders end up as array indices and shift counts, so neither + * can be negative, and a zero collapse order asks for base pages. + */ + if (anon_order < 0 || anon_order > hpage_pmd_order) + ksft_exit_fail_msg("-s takes an order in 0..%d, not %d\n", + hpage_pmd_order, anon_order); + if (collapse_order_set && + (collapse_order <= 0 || collapse_order >= hpage_pmd_order)) + ksft_exit_fail_msg("-c takes an order in 1..%d, not %d\n", + hpage_pmd_order - 1, collapse_order); + argv += optind; argc -= optind; if (argc == 0) { - /* Backwards compatibility */ + /* No arguments: anon under every context */ khugepaged_context = &__khugepaged_context; + mthp_khugepaged_context = &__mthp_khugepaged_context; madvise_context = &__madvise_context; anon_ops = &__anon_ops; return; } - buf = strdup(argv[0]); + buf = argv[0]; token = strsep(&buf, ":"); if (!strcmp(token, "all")) { khugepaged_context = &__khugepaged_context; + mthp_khugepaged_context = &__mthp_khugepaged_context; madvise_context = &__madvise_context; + /* The mTHP context has only anon cases; let other mem_types drop it */ + mthp_context_implied = true; } else if (!strcmp(token, "khugepaged")) { khugepaged_context = &__khugepaged_context; } else if (!strcmp(token, "mthp_khugepaged")) { mthp_khugepaged_context = &__mthp_khugepaged_context; - if (collapse_order <= 0 || collapse_order >= hpage_pmd_order) - usage(); } else if (!strcmp(token, "madvise")) { madvise_context = &__madvise_context; } else { @@ -1254,20 +1607,20 @@ static void parse_test_type(int argc, char **argv) read_write_file_write_ops = &__read_write_file_write_ops; anon_ops = &__anon_ops; shmem_ops = &__shmem_ops; - if (mthp_khugepaged_context) - usage(); } else if (!strcmp(buf, "anon")) { anon_ops = &__anon_ops; } else if (!strcmp(buf, "file")) { read_only_file_ops = &__read_only_file_ops; read_write_file_read_ops = &__read_write_file_read_ops; read_write_file_write_ops = &__read_write_file_write_ops; - if (mthp_khugepaged_context) + if (mthp_khugepaged_context && !mthp_context_implied) usage(); + mthp_khugepaged_context = NULL; } else if (!strcmp(buf, "shmem")) { shmem_ops = &__shmem_ops; - if (mthp_khugepaged_context) + if (mthp_khugepaged_context && !mthp_context_implied) usage(); + mthp_khugepaged_context = NULL; } else { usage(); } @@ -1289,9 +1642,10 @@ struct test_case { struct mem_ops *ops; const char *desc; test_fn fn; + int order; /* mTHP contexts: the collapse order */ }; -#define MAX_TEST_CASES 64 +#define MAX_TEST_CASES 256 static struct test_case test_cases[MAX_TEST_CASES]; static int nr_test_cases; @@ -1304,6 +1658,7 @@ static int nr_test_cases; .ops = o, \ .desc = #t, \ .fn = t, \ + .order = collapse_order, \ }; \ } \ } while (0) @@ -1344,8 +1699,66 @@ int main(int argc, char **argv) parse_test_type(argc, argv); + if (mthp_khugepaged_context) { + unsigned long orders = thp_supported_orders(); + + if (collapse_order_set) { + if (!(orders & (1UL << collapse_order))) + ksft_exit_skip("Order %d is not a supported anon THP order\n", + collapse_order); + if (collapse_order <= anon_order) + ksft_exit_skip("-c %d needs a source order below it, -s says %d\n", + collapse_order, anon_order); + collapse_orders[nr_collapse_orders++] = collapse_order; + } else { + /* + * Every supported order above the source: -s makes the + * fault path hand out folios of that order, so a target + * at or below it has nothing to collapse. + */ + int first = anon_order + 1; + + if (first < MIN_MTHP_ORDER) + first = MIN_MTHP_ORDER; + for (int i = first; i < hpage_pmd_order; i++) { + if (orders & (1UL << i)) + collapse_orders[nr_collapse_orders++] = i; + } + if (!nr_collapse_orders) + ksft_print_msg("mTHP cases skipped: no order above the source\n"); + } + } + + if (mthp_khugepaged_context) { + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_perror("open(/proc/self/pagemap)"); + kpageflags_fd = open("/proc/kpageflags", O_RDONLY); + if (kpageflags_fd < 0) + ksft_exit_fail_perror("open(/proc/kpageflags)"); + } + setbuf(stdout, NULL); + /* + * Without a PMD-order page cache folio the kernel refuses these + * collapses, so there is nothing to test. + */ + if (!(thp_file_supported_orders() & (1UL << hpage_pmd_order))) { + if (shmem_ops) { + ksft_print_msg("no PMD-order page cache folio: skipping shmem\n"); + shmem_ops = NULL; + } + if (read_only_file_ops) { + ksft_print_msg("no PMD-order page cache folio: skipping file\n"); + read_only_file_ops = NULL; + read_write_file_read_ops = NULL; + read_write_file_write_ops = NULL; + } + if (!anon_ops && !shmem_ops && !read_only_file_ops) + ksft_exit_skip("No mem_type left to run\n"); + } + default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1; default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8; default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2; @@ -1363,7 +1776,17 @@ int main(int argc, char **argv) TEST(collapse_full, khugepaged_context, read_write_file_read_ops); TEST(collapse_full, khugepaged_context, read_write_file_write_ops); TEST(collapse_full, khugepaged_context, shmem_ops); - TEST(collapse_full, mthp_khugepaged_context, anon_ops); + for (int i = 0; i < nr_collapse_orders; i++) { + collapse_order = collapse_orders[i]; + TEST(collapse_full, mthp_khugepaged_context, anon_ops); + TEST(collapse_empty, mthp_khugepaged_context, anon_ops); + TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_single_window, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_partial_window, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_max_ptes_none, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_mixed_sources, mthp_khugepaged_context, anon_ops); + } + TEST(collapse_full, madvise_context, anon_ops); TEST(collapse_full, madvise_context, read_only_file_ops); TEST(collapse_full, madvise_context, read_write_file_read_ops); @@ -1371,11 +1794,8 @@ int main(int argc, char **argv) TEST(collapse_full, madvise_context, shmem_ops); TEST(collapse_empty, khugepaged_context, anon_ops); - TEST(collapse_empty, mthp_khugepaged_context, anon_ops); TEST(collapse_empty, madvise_context, anon_ops); - TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops); - TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops); TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops); @@ -1430,6 +1850,9 @@ int main(int argc, char **argv) TEST(collapse_max_ptes_shared, khugepaged_context, anon_ops); TEST(collapse_max_ptes_shared, madvise_context, anon_ops); + TEST(collapse_fork_cow_race, khugepaged_context, anon_ops); + TEST(collapse_fork_cow_race, madvise_context, anon_ops); + TEST(madvise_collapse_existing_thps, madvise_context, anon_ops); TEST(madvise_collapse_existing_thps, madvise_context, read_only_file_ops); TEST(madvise_collapse_existing_thps, madvise_context, read_write_file_read_ops); @@ -1445,7 +1868,15 @@ int main(int argc, char **argv) for (int i = 0; i < nr_test_cases; i++) { struct test_case *t = &test_cases[i]; - ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc, t->ctx->name, t->ops->name); + if (t->ctx == &__mthp_khugepaged_context) { + collapse_order = t->order; + ksft_print_msg("\n# Run test: %s (%s:%s, order %d)\n", + t->desc, t->ctx->name, t->ops->name, + t->order); + } else { + ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc, + t->ctx->name, t->ops->name); + } t->fn(t->ctx, t->ops); } diff --git a/tools/testing/selftests/mm/khugepaged_race.c b/tools/testing/selftests/mm/khugepaged_race.c new file mode 100644 index 000000000000..abb0678522cd --- /dev/null +++ b/tools/testing/selftests/mm/khugepaged_race.c @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Race collapse against faults, GUP pins, fork, mremap and MADV_DONTNEED + * over the same ranges. A racing page must read as its pattern or as + * zero, never anything else; the kernel's own assertions in dmesg are the + * other half of the check. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" +#include "vm_util.h" +#include +#include "../../../../mm/gup_test.h" + +#ifndef FOLL_WRITE +#define FOLL_WRITE 0x01 +#endif + +#define BASE_ADDR ((void *)(1UL << 30)) +#define PASS_TIMEOUT_S 30 + +/* + * PMD-sized areas the racing threads share, plus one for the mremap + * thread. -a shrinks it where a PMD is 512M. + */ +#define DEFAULT_SHARED_AREAS 3 +static int nr_shared_areas; +static int nr_areas; + +static unsigned long hpage_pmd_size; +static unsigned long page_size; +/* nr_areas PMD-sized areas; the last one belongs to the mremap thread */ +static char *region; +static char *mremap_area; +static char *mremap_scratch; +static char *pageout_area; +static size_t pageout_size; +static int gup_fd = -1; +static volatile int stop; +static volatile int corrupted; + +static unsigned int pattern(unsigned long page_idx) +{ + unsigned int val = (unsigned int)page_idx * 2654435761U; + + return val ? val : 1; /* never collides with the zero-fill */ +} + +/* Zero means never written; anything else must be this page's pattern */ +static bool page_is_corrupt(unsigned long page_idx, unsigned int *val) +{ + *val = *(unsigned int *)(region + page_idx * page_size); + + return *val && *val != pattern(page_idx); +} + +static void check_page(unsigned long page_idx) +{ + unsigned int val; + + if (page_is_corrupt(page_idx, &val)) { + corrupted = 1; + ksft_print_msg("Corruption at page %lu: %#x != %#x\n", + page_idx, val, pattern(page_idx)); + } +} + +static unsigned long shared_pages(void) +{ + return nr_shared_areas * hpage_pmd_size / page_size; +} + +static unsigned long rand_page(unsigned int *seed) +{ + return (unsigned long)rand_r(seed) % shared_pages(); +} + +/* Clamp so a range never reaches the mremap thread's area */ +static unsigned long room_from(unsigned long page_idx, unsigned long want) +{ + unsigned long left = shared_pages() - page_idx; + + return want < left ? want : left; +} + +static void *faulter_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + unsigned long page_idx = rand_page(&seed); + + if (rand_r(&seed) & 1) + *(unsigned int *)(region + page_idx * page_size) = + pattern(page_idx); + else + check_page(page_idx); + } + return NULL; +} + +static void *dontneed_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + unsigned long page_idx = rand_page(&seed); + unsigned long nr = 1UL << (rand_r(&seed) % 6); /* 1..32 pages */ + + /* + * Now and then zap a whole PMD-aligned area: only a zap that + * covers the full table frees the table itself (CONFIG_PT_RECLAIM). + */ + if (!(rand_r(&seed) % 64)) { + unsigned long area = page_idx / + (hpage_pmd_size / page_size); + + madvise(region + area * hpage_pmd_size, + hpage_pmd_size, MADV_DONTNEED); + } else { + madvise(region + page_idx * page_size, + room_from(page_idx, nr) * page_size, + MADV_DONTNEED); + } + usleep(rand_r(&seed) % 500); + } + return NULL; +} + +static void *pinner_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + struct gup_test gup = {}; + unsigned long page_idx = rand_page(&seed); + unsigned long nr = room_from(page_idx, 16); + + gup.addr = (unsigned long)(region + page_idx * page_size); + gup.size = nr * page_size; + gup.nr_pages_per_call = nr; + gup.gup_flags = FOLL_WRITE; + /* Racing MADV_DONTNEED makes transient failures expected */ + ioctl(gup_fd, PIN_FAST_BENCHMARK, &gup); + usleep(rand_r(&seed) % 200); + } + return NULL; +} + +static void *forker_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + pid_t pid = fork(); + + if (pid == 0) { + unsigned int val; + int bad = 0; + + /* + * No stdio in the child: a thread may hold stdout's + * lock across the fork, and printing under it hangs. + */ + for (int i = 0; i < 16; i++) + bad |= page_is_corrupt(rand_page(&seed), &val); + _exit(bad); + } + if (pid > 0) { + int wstatus; + + if (waitpid(pid, &wstatus, 0) < 0) + ksft_exit_fail_perror("waitpid()"); + /* A child killed on the read counts too */ + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus)) + corrupted = 1; + } + usleep(rand_r(&seed) % 2000); + } + return NULL; +} + +static void *mremapper_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + void *p; + + p = mremap(mremap_area, hpage_pmd_size, hpage_pmd_size, + MREMAP_MAYMOVE | MREMAP_FIXED, mremap_scratch); + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap() away"); + for (int i = 0; i < 8; i++) + mremap_scratch[(rand_r(&seed) % + (hpage_pmd_size / page_size)) * page_size] = 1; + p = mremap(mremap_scratch, hpage_pmd_size, hpage_pmd_size, + MREMAP_MAYMOVE | MREMAP_FIXED, mremap_area); + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap() back"); + /* The move back unmapped the scratch address: claim it again */ + if (mmap(mremap_scratch, hpage_pmd_size, PROT_NONE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, + -1, 0) != (void *)mremap_scratch) + ksft_exit_fail_perror("mmap() mremap scratch"); + usleep(rand_r(&seed) % 2000); + } + return NULL; +} + +/* + * Swap traffic and LRU churn on a region nothing else writes, so a page's + * pattern must survive the trip through swap exactly. + */ +static void *pageout_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + unsigned long nr = pageout_size / page_size; + unsigned long i; + + for (i = 0; i < nr; i++) + *(unsigned int *)(pageout_area + i * page_size) = pattern(i); + + while (!stop) { + madvise(pageout_area, pageout_size, MADV_PAGEOUT); + for (i = 0; i < nr && !stop; i++) { + unsigned int val = *(unsigned int *)(pageout_area + + i * page_size); + + if (val != pattern(i)) { + corrupted = 1; + ksft_print_msg("Pageout corruption at page %lu: %#x != %#x\n", + i, val, pattern(i)); + } + } + usleep(rand_r(&seed) % 2000); + } + return NULL; +} + +/* Compaction migrates the collapse sources while they are being gathered */ +static void *compactor_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + int fd = open("/proc/sys/vm/compact_memory", O_WRONLY); + + if (fd < 0) { + ksft_print_msg("No compact_memory; compactor idle\n"); + return NULL; + } + while (!stop) { + if (write(fd, "1", 1) < 0) + break; + usleep(10000 + rand_r(&seed) % 100000); + } + close(fd); + return NULL; +} + +static bool swap_available(void) +{ + char line[256]; + int lines = 0; + FILE *fp = fopen("/proc/swaps", "r"); + + if (!fp) + return false; + while (fgets(line, sizeof(line), fp)) + lines++; + fclose(fp); + return lines > 1; +} + +static unsigned long now_ms(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000UL + tv.tv_usec / 1000; +} + +static void usage(void) +{ + fprintf(stderr, + "Usage: khugepaged_race [-d seconds] [-m stepped|free|madvise] [-a areas] [-t mask]\n" + "\tWithout -m, every mode runs in turn.\n" + "\t-d: seconds per mode (default 5)\n" + "\t-a: number of shared PMD-sized playground areas (default 3)\n" + "\t-t: bitmask of racing threads to start, for bisecting a failure\n"); + exit(1); +} + +int main(int argc, char **argv) +{ + static const char * const thread_names[] = { + "faulter", "faulter2", "dontneed", "pinner", "forker", + "mremapper", "pageout", "compactor", + }; + void *(*const thread_fns[])(void *) = { + faulter_fn, faulter_fn, dontneed_fn, pinner_fn, forker_fn, + mremapper_fn, pageout_fn, compactor_fn, + }; + enum { T_FAULTER, T_FAULTER2, T_DONTNEED, T_PINNER, T_FORKER, + T_MREMAPPER, T_PAGEOUT, T_COMPACTOR }; + const unsigned long pageout_bit = 1UL << T_PAGEOUT; + const unsigned long compactor_bit = 1UL << T_COMPACTOR; + const int nr_threads = ARRAY_SIZE(thread_names); + pthread_t threads[ARRAY_SIZE(thread_names)]; + static const char * const all_modes[] = { "stepped", "free", "madvise" }; + static const bool occupancies[] = { false, true }; /* strict, holes */ + static const bool pressures[] = { false, true }; /* quiet, under pressure */ + const int nr_occupancies = ARRAY_SIZE(occupancies); + const int nr_pressures = ARRAY_SIZE(pressures); + const char *one_mode[1]; + const char * const *modes = all_modes; + int nr_modes = ARRAY_SIZE(all_modes); + const char *mode_arg = NULL; + struct thp_settings settings; + unsigned long end_ms; + int duration_s = 5; + unsigned long thread_mask = ~0UL; + unsigned long base_mask; + bool have_swap; + char label[64]; + int nr_areas_arg = 0; + unsigned long i; + int steps = 0; + int opt; + + while ((opt = getopt(argc, argv, "a:d:m:t:h")) != -1) { + switch (opt) { + case 'a': + nr_areas_arg = atoi(optarg); + break; + case 'd': + duration_s = atoi(optarg); + break; + case 'm': + mode_arg = optarg; + break; + case 't': + thread_mask = strtoul(optarg, NULL, 0); + break; + default: + usage(); + } + } + + if (mode_arg) { + if (strcmp(mode_arg, "stepped") && strcmp(mode_arg, "free") && + strcmp(mode_arg, "madvise")) + usage(); + one_mode[0] = mode_arg; + modes = one_mode; + nr_modes = 1; + } + + ksft_print_header(); + if (!thp_available()) + ksft_exit_skip("Transparent Hugepages not available\n"); + + page_size = getpagesize(); + hpage_pmd_size = read_pmd_pagesize(); + if (!hpage_pmd_size) + ksft_exit_fail_msg("Reading PMD pagesize failed\n"); + + gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); + if (gup_fd < 0) + ksft_exit_skip("/sys/kernel/debug/gup_test requires CONFIG_GUP_TEST and root\n"); + + nr_shared_areas = nr_areas_arg > 0 ? nr_areas_arg : DEFAULT_SHARED_AREAS; + nr_areas = nr_shared_areas + 1; + + /* + * MREMAP_FIXED unmaps whatever is in the way without saying so, so + * claim the mremap thread's scratch address up front. + */ + mremap_scratch = (char *)BASE_ADDR + 2 * nr_areas * hpage_pmd_size; + if (mmap(mremap_scratch, hpage_pmd_size, PROT_NONE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, + -1, 0) != (void *)mremap_scratch) + ksft_exit_fail_perror("mmap() mremap scratch"); + + base_mask = thread_mask; + have_swap = swap_available(); + if (!have_swap) + /* No swap, no anon reclaim: compaction-only pressure */ + ksft_print_msg("no swap: the pageout thread is not started\n"); + + ksft_set_plan(nr_modes * nr_occupancies * nr_pressures); + + thp_save_settings(); + thp_read_settings(&settings); + + /* Base of the settings stack; the bottom entry is never popped */ + thp_push_settings(&settings); + + for (int run = 0; run < nr_modes * nr_occupancies * nr_pressures; run++) { + int rem = run % (nr_occupancies * nr_pressures); + const char *mode = modes[run / (nr_occupancies * nr_pressures)]; + bool holes = occupancies[rem / nr_pressures]; + bool pressure = pressures[rem % nr_pressures]; + + snprintf(label, sizeof(label), "%s/%s%s", mode, + holes ? "holes" : "strict", pressure ? "/pressure" : ""); + if (corrupted) { + /* Memory is suspect; the rest would prove nothing */ + ksft_test_result_skip("%s: skipped after corruption\n", label); + continue; + } + + thread_mask = base_mask; + if (!pressure) + thread_mask &= ~(pageout_bit | compactor_bit); + else if (!have_swap) + thread_mask &= ~pageout_bit; + + thp_read_settings(&settings); + settings.thp_enabled = THP_MADVISE; + settings.thp_defrag = THP_DEFRAG_ALWAYS; + settings.shmem_enabled = SHMEM_NEVER; + settings.khugepaged.defrag = 1; + settings.khugepaged.scan_sleep_millisecs = + strcmp(mode, "free") ? 1000 : 0; + settings.khugepaged.alloc_sleep_millisecs = 10; + + /* + * mTHP collapse honours only 0 or HPAGE_PMD_NR - 1 here. The two + * ends race different paths: a strict window has every PTE + * present, a hole-heavy one is mostly zero-filled. + */ + settings.khugepaged.max_ptes_none = holes ? + (hpage_pmd_size / page_size) - 1 : 0; + /* One wake, one pass: the playground plus the forked children's copies */ + settings.khugepaged.pages_to_scan = + nr_areas * (hpage_pmd_size / page_size) * 8; + for (i = 0; i < NR_ORDERS; i++) { + if (thp_supported_orders() & (1UL << i)) + settings.hugepages[i].enabled = THP_INHERIT; + } + thp_push_settings(&settings); + + region = mmap(BASE_ADDR, nr_areas * hpage_pmd_size, + PROT_READ | PROT_WRITE, MAP_ANONYMOUS | + MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0); + if (region != BASE_ADDR) + ksft_exit_fail_perror("mmap() playground"); + mremap_area = region + nr_shared_areas * hpage_pmd_size; + + if (thread_mask & pageout_bit) { + /* Enough to drive real reclaim without swamping a small guest */ + pageout_size = 4 * hpage_pmd_size; + if (pageout_size < 16UL << 20) + pageout_size = 16UL << 20; + if (pageout_size > 64UL << 20) + pageout_size = 64UL << 20; + pageout_area = mmap(NULL, pageout_size, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (pageout_area == MAP_FAILED) + ksft_exit_fail_perror("mmap() pageout area"); + } + + /* Populate so the first pass has something to collapse */ + for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) + *(unsigned int *)(region + i * page_size) = pattern(i); + memset(mremap_area, 1, hpage_pmd_size); + if (madvise(region, nr_areas * hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + + for (i = 0; i < nr_threads; i++) { + if (!(thread_mask & (1UL << i))) { + threads[i] = 0; + continue; + } + if (pthread_create(&threads[i], NULL, thread_fns[i], + (void *)(i + 1))) + ksft_exit_fail_perror(thread_names[i]); + } + + end_ms = now_ms() + duration_s * 1000UL; + if (!strcmp(mode, "stepped")) { + while (now_ms() < end_ms && !corrupted) { + if (!khugepaged_full_pass(PASS_TIMEOUT_S)) + ksft_exit_fail_msg("khugepaged pass timed out\n"); + steps++; + } + } else if (!strcmp(mode, "free")) { + while (now_ms() < end_ms && !corrupted) + usleep(100 * 1000); + } else { /* madvise */ + while (now_ms() < end_ms && !corrupted) { + for (i = 0; i < nr_shared_areas; i++) { + madvise(region + i * hpage_pmd_size, + hpage_pmd_size, MADV_COLLAPSE); + } + madvise(region, nr_shared_areas * hpage_pmd_size, + MADV_DONTNEED); + steps++; + } + } + + stop = 1; + for (i = 0; i < nr_threads; i++) { + if (threads[i]) + pthread_join(threads[i], NULL); + } + + for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) + check_page(i); + + ksft_test_result(!corrupted, "%s: %ds, %d steps, no corruption\n", + label, duration_s, steps); + + /* The next mode maps the same fixed address with its own settings */ + munmap(region, nr_areas * hpage_pmd_size); + if (pageout_area) { + munmap(pageout_area, pageout_size); + pageout_area = NULL; + } + thp_pop_settings(); + stop = 0; + steps = 0; + } + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/khugepaged_sync_check.c b/tools/testing/selftests/mm/khugepaged_sync_check.c new file mode 100644 index 000000000000..28a9b1ff5d44 --- /dev/null +++ b/tools/testing/selftests/mm/khugepaged_sync_check.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Check that khugepaged_full_pass() drives khugepaged in step: one barrier + * over one prepared window must collapse it with exactly one collapse + * attempt attributed to its source pages, step after step. + * + * scan_sleep_millisecs is a minute so that a step which slept instead of + * being woken blows the budget. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" +#include "vm_util.h" +#include + +#define BASE_ADDR ((void *)(1UL << 30)) +/* Smallest order khugepaged considers */ +#define TARGET_ORDER 2 +#define NR_ITERATIONS 5 +#define PASS_TIMEOUT_S 30 + +static int pagemap_fd; +static int kpageflags_fd; +static int trace_events_fd = -1; +static unsigned long hpage_pmd_size; + +/* + * The events are system-wide: switch them off however the test ends, + * including from inside a helper that gives up. + */ +static void trace_events_off(void) +{ + if (trace_events_fd >= 0) + tracing_events_enable(trace_events_fd, false); +} + +/* Count the isolate events whose scan_pfn is one of the window's source PFNs */ +static int count_attributed(unsigned long *pfns, int nr_pfns, + unsigned int order) +{ + char line[1024]; + int count = 0; + FILE *fp; + + fp = tracing_open_trace(); + if (!fp) + ksft_exit_fail_msg("Cannot open trace buffer\n"); + + while (fgets(line, sizeof(line), fp)) { + unsigned long val; + unsigned int ord; + char *s, *o; + int i; + + s = strstr(line, "mm_collapse_huge_page_isolate:"); + if (!s) + continue; + if (sscanf(s, "mm_collapse_huge_page_isolate: scan_pfn=0x%lx", + &val) != 1) + continue; + o = strstr(s, "order="); + if (!o || sscanf(o, "order=%u", &ord) != 1 || ord != order) + continue; + for (i = 0; i < nr_pfns; i++) { + if (val == pfns[i]) { + count++; + break; + } + } + } + fclose(fp); + return count; +} + +static void one_step(int iteration) +{ + const size_t window = getpagesize() << TARGET_ORDER; + const int nr_pages = 1 << TARGET_ORDER; + unsigned long pfns[1 << TARGET_ORDER]; + bool collapsed, passed; + int attributed; + char *p; + int i; + + p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0); + if (p != BASE_ADDR) + ksft_exit_fail_perror("mmap() window"); + + for (i = 0; i < nr_pages; i++) { + p[i * getpagesize()] = i + 1; + pfns[i] = pagemap_get_pfn(pagemap_fd, p + i * getpagesize()); + if (pfns[i] == -1UL) + ksft_exit_fail_msg("Source page not present\n"); + } + + /* Clear before enabling so the buffer holds only this step's events */ + if (tracing_clear_trace()) + ksft_exit_fail_msg("Cannot clear the trace buffer\n"); + if (tracing_events_enable(trace_events_fd, true)) + ksft_exit_fail_msg("Cannot enable huge_memory events\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + passed = khugepaged_full_pass(PASS_TIMEOUT_S); + + /* Off before anything that can give up: the events are system-wide */ + if (tracing_events_enable(trace_events_fd, false)) + ksft_exit_fail_msg("Cannot disable huge_memory events\n"); + if (!passed) + ksft_exit_fail_msg("khugepaged did not complete a full pass\n"); + + collapsed = is_range_backed_by_order(p, window, TARGET_ORDER, + pagemap_fd, kpageflags_fd); + attributed = count_attributed(pfns, nr_pages, TARGET_ORDER); + + ksft_test_result(collapsed && attributed == 1, + "step %d: window collapsed, %d attributed result(s)\n", + iteration, attributed); + + munmap(p, hpage_pmd_size); +} + +int main(void) +{ + struct thp_settings settings; + int i; + + ksft_print_header(); + + if (!thp_available()) + ksft_exit_skip("Transparent Hugepages not available\n"); + if (!(thp_supported_orders() & (1UL << TARGET_ORDER))) + ksft_exit_skip("Order %d is not a supported anon THP order\n", + TARGET_ORDER); + + hpage_pmd_size = read_pmd_pagesize(); + if (!hpage_pmd_size) + ksft_exit_fail_msg("Reading PMD pagesize failed\n"); + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_perror("open(/proc/self/pagemap)"); + kpageflags_fd = open("/proc/kpageflags", O_RDONLY); + if (kpageflags_fd < 0) + ksft_exit_skip("open(/proc/kpageflags) requires root\n"); + trace_events_fd = tracing_events_open("huge_memory"); + if (trace_events_fd < 0) + ksft_exit_skip("huge_memory events require tracefs and root\n"); + atexit(trace_events_off); + + ksft_set_plan(NR_ITERATIONS); + + thp_save_settings(); + thp_read_settings(&settings); + settings.thp_enabled = THP_MADVISE; + settings.thp_defrag = THP_DEFRAG_ALWAYS; + settings.khugepaged.defrag = 1; + settings.khugepaged.scan_sleep_millisecs = 60 * 1000; + settings.khugepaged.alloc_sleep_millisecs = 60 * 1000; + settings.khugepaged.max_ptes_none = (hpage_pmd_size / getpagesize()) - 1; + /* One wake must complete one full pass; see khugepaged_full_pass() */ + settings.khugepaged.pages_to_scan = 1UL << 24; + for (i = 0; i < NR_ORDERS; i++) + settings.hugepages[i].enabled = THP_NEVER; + settings.hugepages[TARGET_ORDER].enabled = THP_INHERIT; + /* Base of the settings stack; the bottom entry is never popped */ + thp_push_settings(&settings); + + for (i = 0; i < NR_ITERATIONS; i++) + one_step(i); + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index 5fd7792a0d47..6711f4c61371 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -15,7 +15,6 @@ #include "kselftest.h" #include #include "vm_util.h" -#include "hugepage_settings.h" #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" #define KSM_FP(s) (KSM_SYSFS_PATH s) diff --git a/tools/testing/selftests/mm/memfd_secret.c b/tools/testing/selftests/mm/memfd_secret.c index c55d84c5e613..aef774be87f1 100644 --- a/tools/testing/selftests/mm/memfd_secret.c +++ b/tools/testing/selftests/mm/memfd_secret.c @@ -145,7 +145,8 @@ static void try_ptrace(int fd, int pipefd[2]) exit(KSFT_FAIL); } - if (ptrace(PTRACE_PEEKDATA, ppid, mem, 0)) + /* PEEKDATA on secret memory must fail, else protection is broken. */ + if (ptrace(PTRACE_PEEKDATA, ppid, mem, 0) == -1) exit(KSFT_PASS); exit(KSFT_FAIL); diff --git a/tools/testing/selftests/mm/memory-failure.c b/tools/testing/selftests/mm/memory-failure.c index f3cb578b1609..135a2e8069ba 100644 --- a/tools/testing/selftests/mm/memory-failure.c +++ b/tools/testing/selftests/mm/memory-failure.c @@ -29,9 +29,14 @@ enum result_type { MADV_HARD_ANON, MADV_HARD_CLEAN_PAGECACHE, MADV_HARD_DIRTY_PAGECACHE, + MADV_HARD_CLEAN_SHMEM, + MADV_HARD_DIRTY_SHMEM, MADV_SOFT_ANON, MADV_SOFT_CLEAN_PAGECACHE, MADV_SOFT_DIRTY_PAGECACHE, + MADV_SOFT_CLEAN_SHMEM, + MADV_SOFT_DIRTY_SHMEM, + READ_ERROR, }; static jmp_buf signal_jmp_buf; @@ -157,17 +162,22 @@ static void check(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure case MADV_HARD_CLEAN_PAGECACHE: case MADV_SOFT_CLEAN_PAGECACHE: case MADV_SOFT_DIRTY_PAGECACHE: - /* It is not expected to receive a SIGBUS signal. */ - ASSERT_EQ(setjmp, 0); - + case MADV_SOFT_DIRTY_SHMEM: /* The page content should remain unchanged. */ ASSERT_TRUE(check_memory(vaddr, self->page_size)); + /* FALLTHROUGH */ + case MADV_HARD_CLEAN_SHMEM: + case MADV_SOFT_CLEAN_SHMEM: + /* It is not expected to receive a SIGBUS signal. */ + ASSERT_EQ(setjmp, 0); /* The backing pfn of addr should have changed. */ ASSERT_NE(pagemap_get_pfn(self->pagemap_fd, vaddr), self->pfn); break; case MADV_HARD_ANON: case MADV_HARD_DIRTY_PAGECACHE: + case MADV_HARD_DIRTY_SHMEM: + case READ_ERROR: /* The SIGBUS signal should have been received. */ ASSERT_EQ(setjmp, 1); @@ -263,6 +273,20 @@ static int prepare_file(const char *fname, unsigned long size) return fd; } +static int prepare_shmem(const char *fname, unsigned long size) +{ + int fd; + + fd = memfd_create(fname, 0); + if (fd < 0) + return -1; + if (ftruncate(fd, size) < 0) { + close(fd); + return -1; + } + return fd; +} + /* Borrowed from mm/gup_longterm.c. */ static int get_fs_type(int fd) { @@ -365,4 +389,88 @@ TEST_F(memory_failure, dirty_pagecache) ASSERT_EQ(close(fd), 0); } +TEST_F(memory_failure, dirty_shmem) +{ + int fd; + char *addr; + int ret; + + fd = prepare_shmem("shmem-file", self->page_size); + if (fd < 0) + SKIP(return, "failed to open test shmem-file.\n"); + + addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + close(fd); + SKIP(return, "mmap failed, not enough memory.\n"); + } + memset(addr, 0xce, self->page_size); + + prepare(_metadata, self, addr); + + ret = sigsetjmp(signal_jmp_buf, 1); + if (!ret && !self->injection_attempted) { + self->injection_attempted = true; + ASSERT_EQ(variant->inject(self, addr), 0); + } + + if (variant->type == MADV_HARD) { + check(_metadata, self, addr, MADV_HARD_DIRTY_SHMEM, ret); + ret = sigsetjmp(signal_jmp_buf, 1); + if (ret == 0) + FORCE_READ(*addr); + check(_metadata, self, addr, READ_ERROR, ret); + } else { + check(_metadata, self, addr, MADV_SOFT_DIRTY_SHMEM, ret); + } + + ASSERT_EQ(munmap(addr, self->page_size), 0); + + ASSERT_EQ(close(fd), 0); +} + +TEST_F(memory_failure, clean_shmem) +{ + int fd; + char *addr; + int ret; + + fd = prepare_shmem("shmem-file", self->page_size); + if (fd < 0) + SKIP(return, "failed to open test shmem-file.\n"); + + addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + close(fd); + SKIP(return, "mmap failed, not enough memory.\n"); + } + FORCE_READ(*addr); + + prepare(_metadata, self, addr); + + ret = sigsetjmp(signal_jmp_buf, 1); + if (!ret && !self->injection_attempted) { + self->injection_attempted = true; + ASSERT_EQ(variant->inject(self, addr), 0); + } + + if (variant->type == MADV_HARD) { + check(_metadata, self, addr, MADV_HARD_CLEAN_SHMEM, ret); + ret = sigsetjmp(signal_jmp_buf, 1); + if (ret == 0) + FORCE_READ(*addr); + check(_metadata, self, addr, READ_ERROR, ret); + } else { + /* Test the address accessability without check_memory(). */ + FORCE_READ(*addr); + check(_metadata, self, addr, MADV_SOFT_CLEAN_SHMEM, ret); + } + + ASSERT_EQ(munmap(addr, self->page_size), 0); + + ASSERT_EQ(close(fd), 0); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index 52b8727b6628..7dd2933417c1 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -1362,6 +1362,101 @@ TEST_F(merge, anon_and_page_offset_mismatch_memfd) ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size); } +TEST_F(merge, merge_map_private_dev_zero_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map two MAP_PRIVATE-/dev/zero VMAs next to one another with offset 0 + * each. + * + * With these being made truly anonymous upon mapping, they will + * merge. If they were file-backed VMAs the page offsets would prevent + * the merge: + * + * |-----||------| |-------------| + * | ptr || ptr2 | -> | ptr | + * |-----||------| |-------------| + */ + ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + close(fd_zero); + ASSERT_NE(ptr, MAP_FAILED); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size); +} + +TEST_F(merge, merge_map_private_dev_zero_faulted_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map a MAP_PRIVATE mapping of /dev/zero with page offset 0, then fault + * it in: + * + * |-------------------------------| + * | faulted | + * |-------------------------------| + */ + ptr = mmap(carveout, 15 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + ASSERT_NE(ptr, MAP_FAILED); + memset(ptr, 'x', 15 * page_size); + + /* + * Unmap the middle: + * + * |---------| |---------| + * | faulted | | faulted | + * |---------| |---------| + */ + ASSERT_EQ(munmap(&ptr[5 * page_size], 5 * page_size), 0); + + /* + * Map in a new unfaulted mapping in the middle with page offset 0 - + * this should merge and would not if it were treated as a file rather + * than pure anon: + * + * |---------|-----------|---------| + * | faulted | unfaulted | faulted | + * |---------|-----------|---------| + */ + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, + fd_zero, 0); + close(fd_zero); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size); +} + TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev) { struct procmap_fd *procmap = &self->procmap; diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index f19d53c69576..d1d0989ed2ca 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -5,7 +5,6 @@ */ #include "kselftest_harness.h" -#include "hugepage_settings.h" #include #include @@ -16,11 +15,11 @@ #include #include #include + #include "vm_util.h" #define TWOMEG (2<<20) #define RUNTIME (20) -#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) HUGETLB_SETUP_DEFAULT_PAGES(1) diff --git a/tools/testing/selftests/mm/mlock-random-test.c b/tools/testing/selftests/mm/mlock-random-test.c index 16294bc7dae6..58772914fd79 100644 --- a/tools/testing/selftests/mm/mlock-random-test.c +++ b/tools/testing/selftests/mm/mlock-random-test.c @@ -71,7 +71,6 @@ int get_proc_locked_vm_size(void) fclose(f); ksft_exit_fail_msg("cannot parse VmLck in /proc/self/status: %s\n", strerror(errno)); - return -1; } /* diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 131d9d6db867..97abf4713cc5 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -111,18 +111,17 @@ static unsigned long long get_mmap_min_addr(void) return addr; fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); - if (fp == NULL) { - ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", - strerror(errno)); - exit(KSFT_SKIP); - } + if (!fp) + ksft_exit_skip("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", + strerror(errno)); n_matched = fscanf(fp, "%llu", &addr); if (n_matched != 1) { - ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", - strerror(errno)); + int err = errno; + fclose(fp); - exit(KSFT_SKIP); + ksft_exit_skip("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", + strerror(err)); } fclose(fp); @@ -156,6 +155,7 @@ static bool is_range_mapped(FILE *maps_fp, unsigned long start, } } + free(line); return success; } @@ -1164,10 +1164,11 @@ static void run_mremap_test_case(struct test test_case, int *failures, rand_addr); if (remap_time < 0) { - if (test_case.expect_failure) - ksft_test_result_xfail("%s\n\tExpected mremap failure\n", - test_case.name); - else { + if (test_case.expect_failure) { + ksft_print_msg("%s: expected mremap failure\n", + test_case.name); + ksft_test_result_xfail("%s\n", test_case.name); + } else { ksft_test_result_fail("%s\n", test_case.name); *failures += 1; } @@ -1177,11 +1178,13 @@ static void run_mremap_test_case(struct test test_case, int *failures, * was faulted in. */ if (threshold_mb == VALIDATION_NO_THRESHOLD || - test_case.config.region_size <= threshold_mb * _1MB) - ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", - test_case.name, remap_time); - else + test_case.config.region_size <= threshold_mb * _1MB) { + ksft_print_msg("%s: mremap time: %12lldns\n", + test_case.name, remap_time); ksft_test_result_pass("%s\n", test_case.name); + } else { + ksft_test_result_pass("%s\n", test_case.name); + } } } @@ -1250,13 +1253,18 @@ int main(int argc, char **argv) time_t t; FILE *maps_fp; + ksft_print_header(); + + get_mmap_min_addr(); + pattern_seed = (unsigned int) time(&t); if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) exit(EXIT_FAILURE); - ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", - threshold_mb, pattern_seed); + ksft_print_msg("Test configs:\n"); + ksft_print_msg("threshold_mb=%u\n", threshold_mb); + ksft_print_msg("pattern_seed=%u\n", pattern_seed); /* * set preallocated random array according to test configs; see the diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index eadc7159ca5b..03898b4f6cda 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -24,7 +24,6 @@ #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ PAGE_IS_FILE | PAGE_IS_PRESENT | \ @@ -36,15 +35,15 @@ #define TEST_ITERATIONS 100 #define PAGEMAP "/proc/self/pagemap" -int pagemap_fd; -int uffd; -size_t page_size; -size_t hpage_size; -const char *progname; +static int pagemap_fd; +static int uffd; +static size_t page_size; +static size_t hpage_size; +static const char *progname; #define LEN(region) ((region.end - region.start)/page_size) -static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag, +static long pagemap_ioctl(void *start, size_t len, void *vec, size_t vec_len, int flag, int max_pages, long required_mask, long anyof_mask, long excluded_mask, long return_mask) { @@ -65,7 +64,7 @@ static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); } -static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, +static long pagemap_ioc(void *start, size_t len, void *vec, size_t vec_len, int flag, int max_pages, long required_mask, long anyof_mask, long excluded_mask, long return_mask, long *walk_end) { @@ -92,8 +91,7 @@ static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, return ret; } - -int init_uffd(void) +static int init_uffd(void) { struct uffdio_api uffdio_api; @@ -116,7 +114,7 @@ int init_uffd(void) return 0; } -int wp_init(void *addr, long size) +static int wp_init(void *addr, size_t size) { struct uffdio_register uffdio_register; struct uffdio_writeprotect wp; @@ -140,7 +138,7 @@ int wp_init(void *addr, long size) return 0; } -int wp_free(void *addr, long size) +static int wp_free(void *addr, size_t size) { struct uffdio_register uffdio_register; @@ -152,7 +150,7 @@ int wp_free(void *addr, long size) return 0; } -int wp_addr_range(void *addr, int size) +static int wp_addr_range(void *addr, size_t size) { if (pagemap_ioctl(addr, size, NULL, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, @@ -162,7 +160,7 @@ int wp_addr_range(void *addr, int size) return 0; } -void *gethugetlb_mem(int size, int *shmid) +static void *gethugetlb_mem(size_t size, int *shmid) { char *mem; @@ -186,9 +184,10 @@ void *gethugetlb_mem(int size, int *shmid) return mem; } -int userfaultfd_tests(void) +static int userfaultfd_tests(void) { - long mem_size, vec_size, written, num_pages = 16; + size_t mem_size, vec_size, num_pages = 16; + long written; char *mem, *vec; mem_size = num_pages * page_size; @@ -229,9 +228,10 @@ int userfaultfd_tests(void) return 0; } -int get_reads(struct page_region *vec, int vec_size) +static int get_reads(struct page_region *vec, size_t vec_size) { - int i, sum = 0; + size_t i; + int sum = 0; for (i = 0; i < vec_size; i++) sum += LEN(vec[i]); @@ -239,9 +239,9 @@ int get_reads(struct page_region *vec, int vec_size) return sum; } -int sanity_tests_sd(void) +static int sanity_tests_sd(void) { - unsigned long long mem_size, vec_size, i, total_pages = 0; + size_t mem_size, vec_size, i, total_pages = 0; long ret, ret2, ret3; int num_pages = 1000; int total_writes, total_reads, reads, count; @@ -331,7 +331,7 @@ int sanity_tests_sd(void) if (ret < 0) ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); - ksft_test_result((unsigned long long)ret == mem_size/(page_size * 2), + ksft_test_result((size_t)ret == mem_size/(page_size * 2), "%s Repeated pattern of written and non-written pages\n", __func__); /* 4. Repeated pattern of written and non-written pages in parts */ @@ -682,9 +682,9 @@ int sanity_tests_sd(void) return 0; } -int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) +static int base_tests(char *prefix, char *mem, size_t mem_size, int skip) { - unsigned long long vec_size; + size_t vec_size; int written; struct page_region *vec, *vec2; @@ -787,7 +787,7 @@ int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) return 0; } -void *gethugepage(int map_size) +static void *gethugepage(size_t map_size) { int ret; char *map; @@ -805,13 +805,13 @@ void *gethugepage(int map_size) return map; } -int hpage_unit_tests(void) +static int hpage_unit_tests(void) { char *map; int ret, ret2; size_t num_pages = 10; - unsigned long long map_size = hpage_size * num_pages; - unsigned long long vec_size = map_size/page_size; + size_t map_size = hpage_size * num_pages; + size_t vec_size = map_size/page_size; struct page_region *vec, *vec2; vec = calloc(vec_size, sizeof(struct page_region)); @@ -999,11 +999,12 @@ int hpage_unit_tests(void) return 0; } -int unmapped_region_tests(void) +static int unmapped_region_tests(void) { void *start = (void *)0x10000000; - int written, len = 0x00040000; - long vec_size = len / page_size; + int written; + size_t len = 0x00040000; + size_t vec_size = len / page_size; struct page_region *vec = calloc(vec_size, sizeof(struct page_region)); if (!vec) ksft_exit_fail_msg("error nomem\n"); @@ -1072,7 +1073,7 @@ static void test_simple(void) * with no page table, exercising pagemap_scan_pte_hole(); a base-page range * leaves pte_none entries. */ -static void unpopulated_written_test(const char *name, char *mem, long size, +static void unpopulated_written_test(const char *name, char *mem, size_t size, bool use_thp) { long npages = size / page_size, fast = 0, slow = 0, ret; @@ -1115,7 +1116,7 @@ out: static void unpopulated_scan_test(void) { - long mem_size = 16 * page_size; + size_t mem_size = 16 * page_size; char *mem; mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, @@ -1155,10 +1156,10 @@ static void unpopulated_thp_scan_test(void) munmap(area, 2 * hpage_size); } -int sanity_tests(void) +static int sanity_tests(void) { - unsigned long long mem_size, vec_size; - long ret, fd, i, buf_size, nr_pages; + size_t mem_size, vec_size, i, buf_size; + long ret, fd, nr_pages; struct page_region *vec; char *mem, *fmem; struct stat sbuf; @@ -1327,7 +1328,7 @@ int sanity_tests(void) return 0; } -int mprotect_tests(void) +static int mprotect_tests(void) { int ret; char *mem, *mem2; @@ -1447,7 +1448,7 @@ static ssize_t get_dirty_pages_reset(char *mem, unsigned int count, return cnt; } -void *thread_proc(void *mem) +static void *thread_proc(void *mem) { int *m = mem; long curr_faults, faults; @@ -1487,7 +1488,7 @@ void *thread_proc(void *mem) return NULL; } -static void transact_test(int page_size) +static void transact_test(void) { unsigned int i, count, extra_pages; unsigned int c; @@ -1580,11 +1581,11 @@ static void transact_test(int page_size) extra_thread_faults); } -void zeropfn_tests(void) +static void zeropfn_tests(void) { - unsigned long long mem_size; + size_t mem_size, i; struct page_region vec; - int i, ret; + int ret; char *mmap_mem, *mem; /* Test with normal memory */ @@ -1642,8 +1643,8 @@ void zeropfn_tests(void) int main(int __attribute__((unused)) argc, char *argv[]) { - int shmid, buf_size, fd, i, ret; - unsigned long long mem_size; + int shmid, fd, ret; + size_t mem_size, buf_size, i; char *mem, *map, *fmem; struct stat sbuf; @@ -1651,6 +1652,9 @@ int main(int __attribute__((unused)) argc, char *argv[]) ksft_print_header(); + page_size = getpagesize(); + hpage_size = read_pmd_pagesize(); + if (init_uffd()) ksft_exit_skip("Failed to initialize userfaultfd\n"); @@ -1659,9 +1663,6 @@ int main(int __attribute__((unused)) argc, char *argv[]) ksft_set_plan(119); - page_size = getpagesize(); - hpage_size = read_pmd_pagesize(); - pagemap_fd = open(PAGEMAP, O_RDONLY); if (pagemap_fd < 0) ksft_exit_fail_msg("Failed to open " PAGEMAP "\n"); @@ -1821,7 +1822,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) mprotect_tests(); /* 13. Transact test */ - transact_test(page_size); + transact_test(); /* 14. Sanity testing */ sanity_tests(); diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index 46a8a1878dc1..9b949951743d 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -114,13 +114,6 @@ void record_pkey_malloc(void *ptr, long size, int prot); #define PKEY_MASK (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE) #endif -/* - * FIXME: Remove once the generic PKEY_UNRESTRICTED definition is merged. - */ -#ifndef PKEY_UNRESTRICTED -#define PKEY_UNRESTRICTED 0x0 -#endif - #ifndef set_pkey_bits static inline u64 set_pkey_bits(u64 reg, int pkey, u64 flags) { diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index c218d0510a2a..f9c728ba96a5 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -543,11 +543,12 @@ static void (*pkey_tests[])(void) = { int main(int argc, char *argv[]) { ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(pkey_tests)); if (!is_pkeys_supported()) ksft_exit_skip("pkeys not supported\n"); + ksft_set_plan(ARRAY_SIZE(pkey_tests)); + for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { tracing_on(); (*pkey_tests[test_nr])(); @@ -555,5 +556,4 @@ int main(int argc, char *argv[]) } ksft_finished(); - return 0; } diff --git a/tools/testing/selftests/mm/prctl_thp_disable.c b/tools/testing/selftests/mm/prctl_thp_disable.c index 82c6e96ea6eb..f9ec1408a6e3 100644 --- a/tools/testing/selftests/mm/prctl_thp_disable.c +++ b/tools/testing/selftests/mm/prctl_thp_disable.c @@ -14,7 +14,6 @@ #include #include "kselftest_harness.h" -#include "hugepage_settings.h" #include "vm_util.h" #ifndef PR_THP_DISABLE_EXCEPT_ADVISED diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index ae6e1530b354..b7882ab97683 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -45,8 +45,8 @@ #include #include #include +#include -#include "hugepage_settings.h" #include "pkey-helpers.h" u64 shadow_pkey_reg; diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index d09f9f6a384e..a1b45a3dedae 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -96,26 +96,44 @@ separated by spaces: example: ./run_vmtests.sh -t "hmm mmap ksm" EOF - exit 0 } RUN_ALL=false RUN_DESTRUCTIVE=false TAP_PREFIX="# " +VM_SELFTEST_ITEMS="default" + while getopts "aht:nd" OPT; do case ${OPT} in "a") RUN_ALL=true ;; - "h") usage ;; + "h") usage; exit 0 ;; "t") VM_SELFTEST_ITEMS=${OPTARG} ;; "n") TAP_PREFIX= ;; "d") RUN_DESTRUCTIVE=true ;; + "?") exit 1 ;; esac done shift $((OPTIND -1)) -# default behavior: run all tests -VM_SELFTEST_ITEMS=${VM_SELFTEST_ITEMS:-default} +# Normalize whitespace so validation and test_selected() use the same names. +read -r -a selected_categories <<< "${VM_SELFTEST_ITEMS//$'\n'/ }" +VM_SELFTEST_ITEMS="${selected_categories[*]}" +if [ -z "$VM_SELFTEST_ITEMS" ]; then + echo "No test categories specified" >&2 + exit 1 +fi + +if [ "$VM_SELFTEST_ITEMS" != "default" ]; then + # Keep the documented category list as the source of valid names. + valid_categories=$(usage | sed -n 's/^- //p') + for category in "${selected_categories[@]}"; do + if ! grep -Fxq -- "$category" <<< "$valid_categories"; then + echo "Unknown test category: $category" >&2 + exit 1 + fi + done +fi test_selected() { if [ "$VM_SELFTEST_ITEMS" == "default" ]; then @@ -130,30 +148,6 @@ test_selected() { fi } -run_gup_matrix() { - # -t: thp=on, -T: thp=off, -H: hugetlb=on - local hugetlb_mb=256 - - for huge in -t -T "-H -m $hugetlb_mb"; do - # -u: gup-fast, -U: gup-basic, -a: pin-fast, -b: pin-basic, -L: pin-longterm - for test_cmd in -u -U -a -b -L; do - # -w: write=1, -W: write=0 - for write in -w -W; do - # -S: shared - for share in -S " "; do - # -n: How many pages to fetch together? 512 is special - # because it's default thp size (or 2M on x86), 123 to - # just test partial gup when hit a huge in whatever form - for num in "-n 1" "-n 512" "-n 123" "-n -1"; do - CATEGORY="gup_test" run_test ./gup_test \ - $huge $test_cmd $write $share $num - done - done - done - done - done -} - # filter 64bit architectures ARCH64STR="arm64 mips64 parisc64 ppc64 ppc64le riscv64 s390x sparc64 x86_64" if [ -z "$ARCH" ]; then @@ -275,18 +269,7 @@ fi CATEGORY="mmap" run_test ./map_fixed_noreplace -if $RUN_ALL; then - run_gup_matrix -else - # get_user_pages_fast() benchmark - CATEGORY="gup_test" run_test ./gup_test -u -n 1 - CATEGORY="gup_test" run_test ./gup_test -u -n -1 - # pin_user_pages_fast() benchmark - CATEGORY="gup_test" run_test ./gup_test -a -n 1 - CATEGORY="gup_test" run_test ./gup_test -a -n -1 -fi -# Dump pages 0, 19, and 4096, using pin_user_pages: -CATEGORY="gup_test" run_test ./gup_test -ct -F 0x1 0 19 0x1000 +CATEGORY="gup_test" run_test ./gup CATEGORY="gup_test" run_test ./gup_longterm CATEGORY="userfaultfd" run_test ./uffd-unit-tests @@ -352,7 +335,7 @@ CATEGORY="process_madv" run_test ./process_madv CATEGORY="vma_merge" run_test ./merge -if [ -x ./memfd_secret ] +if test_selected "memfd_secret" && [ -x ./memfd_secret ] then if [ -f /proc/sys/kernel/yama/ptrace_scope ]; then (echo 0 > /proc/sys/kernel/yama/ptrace_scope 2>&1) | tap_prefix @@ -390,10 +373,7 @@ then CATEGORY="pkey" run_test ./protection_keys_64 fi -if [ -x ./soft-dirty ] -then - CATEGORY="soft_dirty" run_test ./soft-dirty -fi +CATEGORY="soft_dirty" run_test ./soft-dirty CATEGORY="pagemap" run_test ./pagemap_ioctl @@ -402,6 +382,12 @@ CATEGORY="pfnmap" run_test ./pfnmap # COW tests CATEGORY="cow" run_test ./cow +CATEGORY="thp" run_test ./folio_order_check + +CATEGORY="thp" run_test ./khugepaged_sync_check + +CATEGORY="thp" run_test ./khugepaged_race + CATEGORY="thp" run_test ./khugepaged CATEGORY="thp" run_test ./khugepaged -s 2 @@ -410,18 +396,20 @@ CATEGORY="thp" run_test ./khugepaged all:shmem CATEGORY="thp" run_test ./khugepaged -s 4 all:shmem -CATEGORY="thp" run_test ./khugepaged -c 4 mthp_khugepaged:anon - # Try to create XFS if not provided if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then if test_selected "thp"; then if grep xfs /proc/filesystems &>/dev/null; then XFS_IMG=$(mktemp /tmp/xfs_img_XXXXXX) - SPLIT_HUGE_PAGE_TEST_XFS_PATH=$(mktemp -d /tmp/xfs_dir_XXXXXX) + XFS_DIR=$(mktemp -d /tmp/xfs_dir_XXXXXX) truncate -s 314572800 ${XFS_IMG} - mkfs.xfs -q ${XFS_IMG} - mount -o loop ${XFS_IMG} ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} - MOUNTED_XFS=1 + if mkfs.xfs -q ${XFS_IMG} && mount -t xfs -o loop ${XFS_IMG} ${XFS_DIR}; then + SPLIT_HUGE_PAGE_TEST_XFS_PATH=${XFS_DIR} + MOUNTED_XFS=1 + else + rmdir ${XFS_DIR} + rm -f ${XFS_IMG} + fi fi fi fi diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index 5f278913c4d7..7f649b673355 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -9,7 +9,6 @@ #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #define PAGEMAP_FILE_PATH "/proc/self/pagemap" #define TEST_ITERATIONS 10000 diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 86a603692826..c5d96a4b1db3 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -21,7 +21,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" uint64_t pagesize; unsigned int pageshift; @@ -42,68 +41,6 @@ const char *kpageflags_proc = "/proc/kpageflags"; int pagemap_fd; int kpageflags_fd; -static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, - int kpageflags_fd) -{ - const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; - const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; - const unsigned long nr_pages = 1UL << order; - unsigned long pfn_head; - uint64_t pfn_flags; - unsigned long pfn; - unsigned long i; - - pfn = pagemap_get_pfn(pagemap_fd, vaddr); - - /* non present page */ - if (pfn == -1UL) - return false; - - if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) - goto fail; - - /* check for order-0 pages */ - if (!order) { - if (pfn_flags & (folio_head_flags | folio_tail_flags)) - return false; - return true; - } - - /* non THP folio */ - if (!(pfn_flags & KPF_THP)) - return false; - - pfn_head = pfn & ~(nr_pages - 1); - - if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) - goto fail; - - /* head PFN has no compound_head flag set */ - if ((pfn_flags & folio_head_flags) != folio_head_flags) - return false; - - /* check all tail PFN flags */ - for (i = 1; i < nr_pages; i++) { - if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) - goto fail; - if ((pfn_flags & folio_tail_flags) != folio_tail_flags) - return false; - } - - /* - * check the PFN after this folio, but if its flags cannot be obtained, - * assume this folio has the expected order - */ - if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) - return true; - - /* If we find another tail page, then the folio is larger. */ - return (pfn_flags & folio_tail_flags) != folio_tail_flags; -fail: - ksft_exit_fail_msg("Failed to get folio info\n"); - return false; -} - static int check_after_split_folio_orders(char *vaddr_start, size_t len, int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) { @@ -146,7 +83,10 @@ static void write_debugfs(const char *fmt, ...) if (ret >= INPUT_MAX) ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); - write_file(SPLIT_DEBUGFS, input, ret + 1); + ret = write_file(SPLIT_DEBUGFS, input, ret + 1); + if (ret) + ksft_exit_fail_msg("write_file(%s): %s\n", SPLIT_DEBUGFS, + strerror(-ret)); } static char *allocate_zero_filled_hugepage(size_t len) @@ -548,7 +488,6 @@ err_out_close: err_out_unlink: unlink(testfile); ksft_exit_fail_msg("Failed to create large pagecache folios\n"); - return -1; } static void split_thp_in_pagecache_to_order_at(size_t fd_size, @@ -711,6 +650,4 @@ int main(int argc, char **argv) free(expected_orders); ksft_finished(); - - return 0; } diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 50d0805b65db..a04f588df780 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -14,7 +14,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #if !defined(MAP_HUGETLB) #define MAP_HUGETLB 0x40000 diff --git a/tools/testing/selftests/mm/transhuge-stress.c b/tools/testing/selftests/mm/transhuge-stress.c index 8eb0c5630e7e..96f72898ebe0 100644 --- a/tools/testing/selftests/mm/transhuge-stress.c +++ b/tools/testing/selftests/mm/transhuge-stress.c @@ -17,7 +17,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" int backing_fd = -1; int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE; diff --git a/tools/testing/selftests/mm/uffd-common.h b/tools/testing/selftests/mm/uffd-common.h index 92a21b97f745..0723843a7626 100644 --- a/tools/testing/selftests/mm/uffd-common.h +++ b/tools/testing/selftests/mm/uffd-common.h @@ -37,7 +37,6 @@ #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) diff --git a/tools/testing/selftests/mm/uffd-wp-mremap.c b/tools/testing/selftests/mm/uffd-wp-mremap.c index c973d6722720..c48eaab8e75c 100644 --- a/tools/testing/selftests/mm/uffd-wp-mremap.c +++ b/tools/testing/selftests/mm/uffd-wp-mremap.c @@ -7,8 +7,8 @@ #include #include #include +#include #include "kselftest.h" -#include "hugepage_settings.h" #include "uffd-common.h" static int pagemap_fd; @@ -347,6 +347,8 @@ int main(int argc, char **argv) struct thp_settings settings; int i, j, plan = 0; + ksft_print_header(); + hugepage_save_settings(true, true); check_uffd_wp_feature_supported(); diff --git a/tools/testing/selftests/mm/va_high_addr_switch.c b/tools/testing/selftests/mm/va_high_addr_switch.c index e24d7ba00b44..5a354a664d1f 100644 --- a/tools/testing/selftests/mm/va_high_addr_switch.c +++ b/tools/testing/selftests/mm/va_high_addr_switch.c @@ -11,7 +11,6 @@ #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" /* * The hint addr value is used to allocate addresses diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 4821a3563036..a0ab78ceedb1 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -490,6 +490,152 @@ int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags) return 0; } +bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, + int kpageflags_fd) +{ + const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; + const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; + const unsigned long nr_pages = 1UL << order; + unsigned long pfn_head; + uint64_t pfn_flags; + unsigned long pfn; + unsigned long i; + + pfn = pagemap_get_pfn(pagemap_fd, vaddr); + + /* non present page */ + if (pfn == -1UL) + return false; + + if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) + goto fail; + + /* check for order-0 pages */ + if (!order) { + if (pfn_flags & (folio_head_flags | folio_tail_flags)) + return false; + return true; + } + + /* non THP folio */ + if (!(pfn_flags & KPF_THP)) + return false; + + pfn_head = pfn & ~(nr_pages - 1); + + if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) + goto fail; + + /* head PFN has no compound_head flag set */ + if ((pfn_flags & folio_head_flags) != folio_head_flags) + return false; + + /* check all tail PFN flags */ + for (i = 1; i < nr_pages; i++) { + if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) + goto fail; + if ((pfn_flags & folio_tail_flags) != folio_tail_flags) + return false; + } + + /* + * check the PFN after this folio, but if its flags cannot be obtained, + * assume this folio has the expected order + */ + if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) + return true; + + /* If we find another tail page, then the folio is larger. */ + return (pfn_flags & folio_tail_flags) != folio_tail_flags; +fail: + ksft_exit_fail_msg("Failed to get folio info\n"); +} + +/** + * is_range_backed_by_order() - check that a range is backed by @order folios + * @start: start of the range, a multiple of the folio size + * @len: length of the range in bytes, a multiple of the folio size + * @order: the folio order to check for + * @pagemap_fd: open /proc//pagemap of the range's owner + * @kpageflags_fd: open /proc/kpageflags + * + * Every folio-sized, folio-aligned part of the range must map one folio of + * @order, head to tail, with the head at the start of the part. A part + * backed by several smaller folios fails, and so does a folio mapped off + * its natural alignment. + * + * Returns: true if the whole range is backed that way, false otherwise. + */ +bool is_range_backed_by_order(char *start, size_t len, int order, + int pagemap_fd, int kpageflags_fd) +{ + const unsigned long nr_pages = 1UL << order; + const size_t folio_size = nr_pages * psize(); + char *vaddr; + + if ((uintptr_t)start % folio_size || len % folio_size) + return false; + + for (vaddr = start; vaddr < start + len; vaddr += folio_size) { + const unsigned long pfn = pagemap_get_pfn(pagemap_fd, vaddr); + unsigned long i; + + /* Not present, or a tail page */ + if (pfn == -1UL || pfn % nr_pages) + return false; + + for (i = 1; i < nr_pages; i++) { + char *page = vaddr + i * psize(); + + if (pagemap_get_pfn(pagemap_fd, page) != pfn + i) + return false; + } + + if (!is_backed_by_folio(vaddr, order, pagemap_fd, kpageflags_fd)) + return false; + } + + return true; +} + +#define TRACEFS_ROOT "/sys/kernel/tracing" + +/* + * Returns -1 without tracefs or the subsystem. The events are system-wide: + * whoever switches them on has to switch them off again, on every exit path. + */ +int tracing_events_open(const char *subsys) +{ + char path[256]; + + snprintf(path, sizeof(path), TRACEFS_ROOT "/events/%s/enable", + subsys); + return open(path, O_WRONLY); +} + +int tracing_events_enable(int fd, bool enable) +{ + if (pwrite(fd, enable ? "1" : "0", 1, 0) != 1) + return -1; + return 0; +} + +/* Drop what the trace buffer holds so far */ +int tracing_clear_trace(void) +{ + int fd = open(TRACEFS_ROOT "/trace", O_WRONLY | O_TRUNC); + + if (fd < 0) + return -1; + close(fd); + return 0; +} + +FILE *tracing_open_trace(void) +{ + return fopen(TRACEFS_ROOT "/trace", "r"); +} + /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor, uint64_t *ioctls) @@ -885,111 +1031,54 @@ int unpoison_memory(unsigned long pfn) return ret > 0 ? 0 : -errno; } -int read_file(const char *path, char *buf, size_t buflen) -{ - int fd; - ssize_t numread; - - fd = open(path, O_RDONLY); - if (fd == -1) - return 0; - - numread = read(fd, buf, buflen - 1); - if (numread < 1) { - close(fd); - return 0; - } - - buf[numread] = '\0'; - close(fd); - - return (unsigned int) numread; -} - -static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) -{ - int fd, saved_errno; - ssize_t numwritten; - - if (buflen < 2) - ksft_exit_fail_msg("Incorrect buffer len: %zu\n", buflen); - - fd = open(path, O_WRONLY); - if (fd == -1) - ksft_exit_fail_msg("%s open failed: %s\n", path, strerror(errno)); - - numwritten = write(fd, buf, buflen - 1); - saved_errno = errno; - close(fd); - errno = saved_errno; - if (numwritten < 0) { - if (ignore_einval && errno == EINVAL) - return; - ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), - buf, strerror(errno)); - } - if (numwritten != buflen - 1) - ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", - path, (int)(buflen - 1), buf, buflen - 1, numwritten); -} - -void write_file(const char *path, const char *buf, size_t buflen) -{ - __write_file(path, buf, buflen, /* ignore_einval = */ false); -} - -unsigned long read_num(const char *path) -{ - char buf[21]; - - if (!read_file(path, buf, sizeof(buf))) - ksft_exit_fail_perror("read_file()"); - - return strtoul(buf, NULL, 10); -} - -static void __write_num(const char *path, unsigned long num, bool ignore_einval) -{ - char buf[21]; - - sprintf(buf, "%lu", num); - __write_file(path, buf, strlen(buf) + 1, ignore_einval); -} - -void write_num(const char *path, unsigned long num) -{ - return __write_num(path, num, /* ignore_einval = */ false); -} - -void write_num_ignore_einval(const char *path, unsigned long num) -{ - return __write_num(path, num, /* ignore_einval = */ true); -} - static unsigned long shmall, shmmax; void __shm_limits_restore(void) { - if (shmmax) - write_num("/proc/sys/kernel/shmmax", shmmax); - if (shmall) - write_num("/proc/sys/kernel/shmall", shmall); + int ret; + + if (shmmax) { + ret = write_num("/proc/sys/kernel/shmmax", shmmax); + if (ret < 0) + ksft_exit_fail_msg("Failed to restore shmmax: %s\n", + strerror(-ret)); + } + if (shmall) { + ret = write_num("/proc/sys/kernel/shmall", shmall); + if (ret < 0) + ksft_exit_fail_msg("Failed to restore shmall: %s\n", + strerror(-ret)); + } } void shm_limits_prepare(unsigned long length) { unsigned long nr = length / psize(); unsigned long val; + int ret; + + ret = read_num("/proc/sys/kernel/shmmax", &val); + if (ret < 0) + ksft_exit_fail_msg("Failed to read /proc/sys/kernel/shmmax: %s\n", + strerror(-ret)); - val = read_num("/proc/sys/kernel/shmmax"); if (val < length) { - write_num("/proc/sys/kernel/shmmax", length); + ret = write_num("/proc/sys/kernel/shmmax", length); + if (ret < 0) + ksft_exit_fail_msg("Failed to write %lu to /proc/sys/kernel/shmmax: %s\n", + length, strerror(-ret)); shmmax = val; } - val = read_num("/proc/sys/kernel/shmall"); + ret = read_num("/proc/sys/kernel/shmall", &val); + if (ret < 0) + ksft_exit_fail_msg("Failed to read /proc/sys/kernel/shmall: %s\n", + strerror(-ret)); if (val < nr) { - write_num("/proc/sys/kernel/shmall", nr); + ret = write_num("/proc/sys/kernel/shmall", nr); + if (ret < 0) + ksft_exit_fail_msg("Failed to write %lu to /proc/sys/kernel/shmall: %s\n", + nr, strerror(-ret)); shmall = val; } } diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 9a49af88702e..072a6c756c51 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -8,8 +8,12 @@ #include /* _SC_PAGESIZE */ #include "kselftest.h" #include +#include +#include #define BIT_ULL(nr) (1ULL << (nr)) +#define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) + #define PM_SOFT_DIRTY BIT_ULL(55) #define PM_MMAP_EXCLUSIVE BIT_ULL(56) #define PM_UFFD_WP BIT_ULL(57) @@ -97,6 +101,10 @@ int64_t allocate_transhuge(void *ptr, int pagemap_fd); int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags); int gather_folio_orders(char *vaddr_start, size_t len, int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders); +bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, + int kpageflags_fd); +bool is_range_backed_by_order(char *start, size_t len, int order, + int pagemap_fd, int kpageflags_fd); int uffd_register(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor); @@ -113,6 +121,10 @@ int close_procmap(struct procmap_fd *procmap); int write_sysfs(const char *file_path, unsigned long val); int read_sysfs(const char *file_path, unsigned long *val); bool softdirty_supported(void); +int tracing_events_open(const char *subsys); +int tracing_events_enable(int fd, bool enable); +int tracing_clear_trace(void); +FILE *tracing_open_trace(void); static inline int open_self_procmap(struct procmap_fd *procmap_out) { @@ -166,12 +178,6 @@ int unpoison_memory(unsigned long pfn); #define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0) #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1)) -void write_file(const char *path, const char *buf, size_t buflen); -int read_file(const char *path, char *buf, size_t buflen); -unsigned long read_num(const char *path); -void write_num(const char *path, unsigned long num); -void write_num_ignore_einval(const char *path, unsigned long num); - void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index 415eccb70468..bf4c5073f6fc 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -80,6 +80,61 @@ enum maps_file { struct vma_modifier_info; +enum smaps_rollup_stat { + Rss, + Pss, + Pss_Dirty, + Pss_Anon, + Pss_File, + Pss_Shmem, + Shared_Clean, + Shared_Dirty, + Private_Clean, + Private_Dirty, + Referenced, + Anonymous, + KSM, + LazyFree, + AnonHugePages, + ShmemPmdMapped, + FilePmdMapped, + Shared_Hugetlb, + Private_Hugetlb, + Swap, + SwapPss, + Locked, + RollupFieldCount +}; + +static const char *smaps_rollup_stat_names[RollupFieldCount] = { + "Rss", + "Pss", + "Pss_Dirty", + "Pss_Anon", + "Pss_File", + "Pss_Shmem", + "Shared_Clean", + "Shared_Dirty", + "Private_Clean", + "Private_Dirty", + "Referenced", + "Anonymous", + "KSM", + "LazyFree", + "AnonHugePages", + "ShmemPmdMapped", + "FilePmdMapped", + "Shared_Hugetlb", + "Private_Hugetlb", + "Swap", + "SwapPss", + "Locked", +}; + +struct smaps_rollup_stats { + unsigned long values[RollupFieldCount]; +}; + FIXTURE(proc_maps_race) { struct vma_modifier_info *mod_info; @@ -91,6 +146,7 @@ FIXTURE(proc_maps_race) enum maps_file maps_file; int shared_mem_size; int skip_pages; + int rollup_fd; int page_size; int vma_count; bool verbose; @@ -132,12 +188,12 @@ struct vma_modifier_info { void *child_mapped_addr[]; }; -static bool read_page(FIXTURE_DATA(proc_maps_race) *self, +static bool read_page(FIXTURE_DATA(proc_maps_race) *self, int fd, struct page_content *page) { ssize_t bytes_read; - bytes_read = read(self->maps_fd, page->data, self->page_size); + bytes_read = read(fd, page->data, self->page_size); if (bytes_read <= 0) return false; @@ -175,7 +231,7 @@ static int locate_containing_page(FIXTURE_DATA(proc_maps_race) *self, char *curr_pos; char *end_pos; - if (!read_page(self, &self->page1)) + if (!read_page(self, self->maps_fd, &self->page1)) return -1; curr_pos = self->page1.data; @@ -205,10 +261,11 @@ static bool read_two_pages(FIXTURE_DATA(proc_maps_race) *self) return false; for (int i = 0; i < self->skip_pages; i++) - if (!read_page(self, &self->page1)) + if (!read_page(self, self->maps_fd, &self->page1)) return false; - return read_page(self, &self->page1) && read_page(self, &self->page2); + return read_page(self, self->maps_fd, &self->page1) && + read_page(self, self->maps_fd, &self->page2); } static void copy_line(const char *line_start, const char *line_end, @@ -317,6 +374,61 @@ static bool read_boundary_lines(FIXTURE_DATA(proc_maps_race) *self, &first_line->end_addr) == 2; } +static bool parse_smaps_rollup(FIXTURE_DATA(proc_maps_race) *self, + struct smaps_rollup_stats *stats) +{ + unsigned int dev_maj, dev_min, inode; + unsigned long start, end, offs; + unsigned long value; + char name[32], perm[5]; + char *curr_pos; + char *end_pos; + char *line_end; + + if (lseek(self->rollup_fd, 0, SEEK_SET) < 0) + return false; + + if (!read_page(self, self->rollup_fd, &self->page1)) + return false; + + curr_pos = self->page1.data; + end_pos = self->page1.data + self->page1.size; + + line_end = strchr(curr_pos, '\n'); + if (!line_end) + return false; + + if (sscanf(curr_pos, "%lx-%lx %4s %lx %u:%u %u %31s", + &start, &end, perm, &offs, &dev_maj, &dev_min, &inode, name) != 8) + return false; + + if (strcmp(name, "[rollup]")) + return false; + + for (int stat = 0; stat < ARRAY_SIZE(smaps_rollup_stat_names); stat++) { + int len; + + curr_pos = line_end + 1; + if (curr_pos >= end_pos) + return false; + + line_end = strchr(curr_pos, '\n'); + if (!line_end) + return false; + + if (sscanf(curr_pos, "%31s %lu kB", name, &value) != 2) + return false; + + len = strlen(name); + if (name[len - 1] != ':' || strncmp(name, smaps_rollup_stat_names[stat], len - 1)) + return false; + + stats->values[stat] = value; + } + + return true; +} + /* Thread synchronization routines */ static void wait_for_state(struct vma_modifier_info *mod_info, enum test_state state) { @@ -397,6 +509,40 @@ static bool print_boundaries_on(bool condition, const char *title, return condition; } +static void print_smaps_rollup_stats(const char *title, FIXTURE_DATA(proc_maps_race) *self, + struct smaps_rollup_stats *stats) +{ + printf("%s", title); + for (int stat = 0; stat < ARRAY_SIZE(smaps_rollup_stat_names); stat++) + printf("%64s %lu kB\n", smaps_rollup_stat_names[stat], stats->values[stat]); +} + +static bool cmp_smaps_rollup_stat(struct smaps_rollup_stats *s1, + struct smaps_rollup_stats *s2, enum smaps_rollup_stat stat) +{ + return s1->values[stat] == s2->values[stat]; +} + +static bool compare_smaps_rollup(FIXTURE_DATA(proc_maps_race) *self, + struct smaps_rollup_stats *expected, + struct smaps_rollup_stats *actual) +{ + /* + * Clean/dirty metrics might change but Pss-related ones + * should stay constant. + */ + if (cmp_smaps_rollup_stat(expected, actual, Pss) && + cmp_smaps_rollup_stat(expected, actual, Pss_Anon) && + cmp_smaps_rollup_stat(expected, actual, Pss_File) && + cmp_smaps_rollup_stat(expected, actual, Pss_Shmem)) + return true; + + print_smaps_rollup_stats("Expected stats:", self, expected); + print_smaps_rollup_stats("Actual stats:", self, actual); + + return false; +} + static void report_test_start(const char *name, bool verbose) { if (verbose) @@ -572,6 +718,7 @@ FIXTURE_SETUP(proc_maps_race) unsigned long first_map_addr; unsigned long last_map_addr; unsigned long duration_sec; + char rollup_fname[32]; char fname[32]; self->page_size = (unsigned long)sysconf(_SC_PAGESIZE); @@ -649,6 +796,9 @@ FIXTURE_SETUP(proc_maps_race) break; case SMAPS: sprintf(fname, "/proc/%d/smaps", self->pid); + sprintf(rollup_fname, "/proc/%d/smaps_rollup", self->pid); + self->rollup_fd = open(rollup_fname, O_RDONLY); + ASSERT_NE(self->rollup_fd, -1); break; default: ksft_exit_fail(); @@ -711,6 +861,8 @@ FIXTURE_TEARDOWN(proc_maps_race) for (int i = 0; i < self->vma_count; i++) munmap(self->mod_info->child_mapped_addr[i], self->page_size); close(self->maps_fd); + if (self->maps_file == SMAPS) + close(self->rollup_fd); waitpid(self->pid, &status, 0); munmap(self->mod_info, self->shared_mem_size); } @@ -723,6 +875,7 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) struct line_content split_first_line; struct line_content restored_last_line; struct line_content restored_first_line; + struct smaps_rollup_stats orig_stats; wait_for_state(mod_info, SETUP_READY); @@ -736,6 +889,8 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) report_test_start("Tearing from split", self->verbose); ASSERT_TRUE(capture_mod_pattern(self, &split_last_line, &split_first_line, &restored_last_line, &restored_first_line)); + if (self->maps_file == SMAPS) + ASSERT_TRUE(parse_smaps_rollup(self, &orig_stats)); /* Now start concurrent modifications for self->duration_sec */ signal_state(mod_info, TEST_READY); @@ -799,6 +954,11 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) vma_end == self->last_line.end_addr) || (vma_start == split_first_line.start_addr && vma_end == split_first_line.end_addr)); + } else { + struct smaps_rollup_stats stats; + + ASSERT_TRUE(parse_smaps_rollup(self, &stats)); + ASSERT_TRUE(compare_smaps_rollup(self, &orig_stats, &stats)); } clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); @@ -817,6 +977,7 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) struct line_content shrunk_first_line; struct line_content restored_last_line; struct line_content restored_first_line; + struct smaps_rollup_stats orig_stats; wait_for_state(mod_info, SETUP_READY); @@ -830,6 +991,8 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) report_test_start("Tearing from resize", self->verbose); ASSERT_TRUE(capture_mod_pattern(self, &shrunk_last_line, &shrunk_first_line, &restored_last_line, &restored_first_line)); + if (self->maps_file == SMAPS) + ASSERT_TRUE(parse_smaps_rollup(self, &orig_stats)); /* Now start concurrent modifications for self->duration_sec */ signal_state(mod_info, TEST_READY); @@ -880,6 +1043,11 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) ASSERT_TRUE(vma_start == self->last_line.start_addr && (vma_end - vma_start == self->page_size * 3 || vma_end - vma_start == self->page_size)); + } else { + struct smaps_rollup_stats stats; + + ASSERT_TRUE(parse_smaps_rollup(self, &stats)); + ASSERT_TRUE(compare_smaps_rollup(self, &orig_stats, &stats)); } clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); @@ -898,6 +1066,7 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) struct line_content remapped_first_line; struct line_content restored_last_line; struct line_content restored_first_line; + struct smaps_rollup_stats orig_stats; wait_for_state(mod_info, SETUP_READY); @@ -911,6 +1080,8 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) report_test_start("Tearing from remap", self->verbose); ASSERT_TRUE(capture_mod_pattern(self, &remapped_last_line, &remapped_first_line, &restored_last_line, &restored_first_line)); + if (self->maps_file == SMAPS) + ASSERT_TRUE(parse_smaps_rollup(self, &orig_stats)); /* Now start concurrent modifications for self->duration_sec */ signal_state(mod_info, TEST_READY); @@ -963,6 +1134,11 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) vma_end - vma_start == self->page_size * 3) || (vma_start == self->last_line.start_addr + self->page_size && vma_end - vma_start == self->page_size)); + } else { + struct smaps_rollup_stats stats; + + ASSERT_TRUE(parse_smaps_rollup(self, &stats)); + ASSERT_TRUE(compare_smaps_rollup(self, &orig_stats, &stats)); } clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); diff --git a/tools/testing/selftests/proc/proc-self-map-files-001.c b/tools/testing/selftests/proc/proc-self-map-files-001.c index 4209c64283d6..bbca9f9e2743 100644 --- a/tools/testing/selftests/proc/proc-self-map-files-001.c +++ b/tools/testing/selftests/proc/proc-self-map-files-001.c @@ -51,7 +51,7 @@ int main(void) int fd; unsigned long a, b; - fd = open("/dev/zero", O_RDONLY); + fd = open("/proc/self/exe", O_RDONLY); if (fd == -1) return 1; diff --git a/tools/testing/selftests/proc/proc-self-map-files-002.c b/tools/testing/selftests/proc/proc-self-map-files-002.c index e6aa00a183bc..5786cdffbbf6 100644 --- a/tools/testing/selftests/proc/proc-self-map-files-002.c +++ b/tools/testing/selftests/proc/proc-self-map-files-002.c @@ -57,7 +57,7 @@ int main(void) int fd; unsigned long a, b; - fd = open("/dev/zero", O_RDONLY); + fd = open("/proc/self/exe", O_RDONLY); if (fd == -1) return 1; diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 4c58487b764e..16c09dac59d9 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -560,7 +560,6 @@ struct vm_area_struct { vma_flags_t flags; }; -#ifdef CONFIG_PER_VMA_LOCK /* * Can only be written (using WRITE_ONCE()) while holding both: * - mmap_lock (in write mode) @@ -576,7 +575,7 @@ struct vm_area_struct { * slowpath. */ unsigned int vm_lock_seq; -#endif + unsigned int __vm_anon_pgoff_lo; /* @@ -610,10 +609,8 @@ struct vm_area_struct { #ifdef CONFIG_NUMA_BALANCING struct vma_numab_state *numab_state; /* NUMA Balancing state */ #endif -#ifdef CONFIG_PER_VMA_LOCK /* Unstable RCU readers are allowed to read this. */ refcount_t vm_refcnt; -#endif #ifdef CONFIG_64BIT unsigned int __vm_anon_pgoff_hi; #endif @@ -1638,9 +1635,15 @@ static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, const pgoff_t pgoff = __linear_anon_page_index(vma, address); VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); - /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ - if (vma_is_anonymous(vma) && !vma->vm_file) + if (vma_is_anonymous(vma)) VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); return pgoff; } + +extern const struct file_operations zero_fops; + +static inline bool file_is_dev_zero(const struct file *file) +{ + return file && file->f_op == &zero_fops; +} diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c index 4a39c9d50489..8c4826499f40 100644 --- a/tools/testing/vma/shared.c +++ b/tools/testing/vma/shared.c @@ -12,6 +12,15 @@ const struct vm_operations_struct vma_dummy_vm_ops; struct anon_vma dummy_anon_vma; struct task_struct __current; +static int mmap_zero_prepare(struct vm_area_desc *desc) +{ + return 0; +} + +const struct file_operations zero_fops = { + .mmap_prepare = mmap_zero_prepare, +}; + struct vm_area_struct *alloc_vma(struct mm_struct *mm, unsigned long start, unsigned long end, pgoff_t pgoff, vma_flags_t vma_flags) diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb..53e4abe6a63e 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -45,7 +45,112 @@ static bool test_mmap_region_basic(void) return true; } +static bool mmap_region_fill_hole(bool merge) +{ + const vma_flags_t vma_flags = mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT); + vma_flags_t middle_flags = vma_flags; + struct mm_struct mm = {}; + struct vm_area_struct *vma; + unsigned long addr; + int count = 0; + VMA_ITERATOR(vmi, &mm, 0); + + current->mm = &mm; + if (!merge) + vma_flags_set(&middle_flags, VMA_EXEC_BIT); + + /* Map at 0x300000, length 0x3000. */ + addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, 0x300, NULL); + ASSERT_EQ(addr, 0x300000); + + /* Map at 0x306000, length 0x3000, leaving a hole. */ + addr = __mmap_region(NULL, 0x306000, 0x3000, vma_flags, 0x306, NULL); + ASSERT_EQ(addr, 0x306000); + ASSERT_EQ(mm.map_count, 2); + + /* Map at 0x303000, length 0x3000, filling the hole. */ + addr = __mmap_region(NULL, 0x303000, 0x3000, middle_flags, 0x303, NULL); + ASSERT_EQ(addr, 0x303000); + ASSERT_EQ(mm.map_count, merge ? 1 : 3); + + vma_iter_set(&vmi, 0); + for_each_vma(vmi, vma) { + const unsigned long start = 0x300000 + count * 0x3000; + const unsigned long end = merge ? 0x309000 : start + 0x3000; + /* Only the middle VMA in the non-merge case has VMA_EXEC. */ + const bool is_middle_vma = count == 1; + const bool expect_exec_vma = is_middle_vma && !merge; + + ASSERT_EQ(vma->vm_start, start); + ASSERT_EQ(vma->vm_end, end); + ASSERT_EQ(vma_start_pgoff(vma), start >> PAGE_SHIFT); + ASSERT_EQ(vma_start_anon_pgoff(vma), start >> PAGE_SHIFT); + + ASSERT_TRUE(vma_test_all(vma, VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, + VMA_MAYEXEC_BIT)); + ASSERT_EQ(vma_test(vma, VMA_EXEC_BIT), expect_exec_vma); + + count++; + } + + ASSERT_EQ(count, mm.map_count); + + ASSERT_EQ(cleanup_mm(&mm, &vmi), count); + return true; +} + +static bool test_mmap_region_fill_hole_merge(void) +{ + return mmap_region_fill_hole(true); +} + +static bool test_mmap_region_fill_hole_flags_mismatch(void) +{ + return mmap_region_fill_hole(false); +} + +static bool test_pure_anon_dev_zero(void) +{ + const vma_flags_t vma_flags = mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT); + struct file file = { + .f_op = &zero_fops, + }; + struct mm_struct mm = {}; + struct vm_area_struct *vma; + unsigned long addr; + VMA_ITERATOR(vmi, &mm, 0); + + current->mm = &mm; + + /* + * Map a MAP_PRIVATE-/dev/zero mapping at address 0x300000 with a page + * offset of 0x10, which we expect to be reset to the anonymous page + * offset. + */ + addr = __mmap_region(&file, 0x300000, 0x3000, vma_flags, 0x10, NULL); + ASSERT_EQ(addr, 0x300000); + + /* Assert that it truly is an anonymous mapping. */ + vma = vma_lookup(&mm, addr); + ASSERT_NE(vma, NULL); + ASSERT_TRUE(vma_is_anonymous(vma)); + ASSERT_EQ(vma->vm_file, NULL); + ASSERT_EQ(vma->vm_private_data, NULL); + /* Expect anonymous page offsets. */ + ASSERT_EQ(vma->vm_pgoff, 0x300); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0x300); + + cleanup_mm(&mm, &vmi); + return true; +} + static void run_mmap_tests(int *num_tests, int *num_fail) { TEST(mmap_region_basic); + TEST(mmap_region_fill_hole_merge); + TEST(mmap_region_fill_hole_flags_mismatch); + TEST(pure_anon_dev_zero); } diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h index 8a48b231aa7a..54d5c3360aa2 100644 --- a/tools/testing/vma/vma_internal.h +++ b/tools/testing/vma/vma_internal.h @@ -15,7 +15,6 @@ #include #define CONFIG_MMU 1 -#define CONFIG_PER_VMA_LOCK 1 #ifdef __CONCAT #undef __CONCAT