kernel/alloc/
allocator.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! Allocator support.
4//!
5//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide"
6//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the
7//! typical application of the different kernel allocators.
8//!
9//! Reference: <https://docs.kernel.org/core-api/memory-allocation.html>
10
11use super::Flags;
12use core::alloc::Layout;
13use core::ptr;
14use core::ptr::NonNull;
15
16use crate::alloc::{AllocError, Allocator};
17use crate::bindings;
18use crate::pr_warn;
19
20/// The contiguous kernel allocator.
21///
22/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also
23/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific.
24///
25/// For more details see [self].
26pub struct Kmalloc;
27
28/// The virtually contiguous kernel allocator.
29///
30/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel
31/// virtual space. It is typically used for large allocations. The memory allocated with this
32/// allocator is not physically contiguous.
33///
34/// For more details see [self].
35pub struct Vmalloc;
36
37/// The kvmalloc kernel allocator.
38///
39/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon
40/// failure. This allocator is typically used when the size for the requested allocation is not
41/// known and may exceed the capabilities of `Kmalloc`.
42///
43/// For more details see [self].
44pub struct KVmalloc;
45
46/// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment.
47fn aligned_size(new_layout: Layout) -> usize {
48    // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first.
49    let layout = new_layout.pad_to_align();
50
51    // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()`
52    // which together with the slab guarantees means the `krealloc` will return a properly aligned
53    // object (see comments in `kmalloc()` for more information).
54    layout.size()
55}
56
57/// # Invariants
58///
59/// One of the following: `krealloc`, `vrealloc`, `kvrealloc`.
60struct ReallocFunc(
61    unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void,
62);
63
64impl ReallocFunc {
65    // INVARIANT: `krealloc` satisfies the type invariants.
66    const KREALLOC: Self = Self(bindings::krealloc);
67
68    // INVARIANT: `vrealloc` satisfies the type invariants.
69    const VREALLOC: Self = Self(bindings::vrealloc);
70
71    // INVARIANT: `kvrealloc` satisfies the type invariants.
72    const KVREALLOC: Self = Self(bindings::kvrealloc);
73
74    /// # Safety
75    ///
76    /// This method has the same safety requirements as [`Allocator::realloc`].
77    ///
78    /// # Guarantees
79    ///
80    /// This method has the same guarantees as `Allocator::realloc`. Additionally
81    /// - it accepts any pointer to a valid memory allocation allocated by this function.
82    /// - memory allocated by this function remains valid until it is passed to this function.
83    unsafe fn call(
84        &self,
85        ptr: Option<NonNull<u8>>,
86        layout: Layout,
87        old_layout: Layout,
88        flags: Flags,
89    ) -> Result<NonNull<[u8]>, AllocError> {
90        let size = aligned_size(layout);
91        let ptr = match ptr {
92            Some(ptr) => {
93                if old_layout.size() == 0 {
94                    ptr::null()
95                } else {
96                    ptr.as_ptr()
97                }
98            }
99            None => ptr::null(),
100        };
101
102        // SAFETY:
103        // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that
104        //   `ptr` is NULL or valid.
105        // - `ptr` is either NULL or valid by the safety requirements of this function.
106        //
107        // GUARANTEE:
108        // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`.
109        // - Those functions provide the guarantees of this function.
110        let raw_ptr = unsafe {
111            // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed.
112            self.0(ptr.cast(), size, flags.0).cast()
113        };
114
115        let ptr = if size == 0 {
116            crate::alloc::dangling_from_layout(layout)
117        } else {
118            NonNull::new(raw_ptr).ok_or(AllocError)?
119        };
120
121        Ok(NonNull::slice_from_raw_parts(ptr, size))
122    }
123}
124
125// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
126// - memory remains valid until it is explicitly freed,
127// - passing a pointer to a valid memory allocation is OK,
128// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
129unsafe impl Allocator for Kmalloc {
130    #[inline]
131    unsafe fn realloc(
132        ptr: Option<NonNull<u8>>,
133        layout: Layout,
134        old_layout: Layout,
135        flags: Flags,
136    ) -> Result<NonNull<[u8]>, AllocError> {
137        // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`.
138        unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) }
139    }
140}
141
142// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
143// - memory remains valid until it is explicitly freed,
144// - passing a pointer to a valid memory allocation is OK,
145// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
146unsafe impl Allocator for Vmalloc {
147    #[inline]
148    unsafe fn realloc(
149        ptr: Option<NonNull<u8>>,
150        layout: Layout,
151        old_layout: Layout,
152        flags: Flags,
153    ) -> Result<NonNull<[u8]>, AllocError> {
154        // TODO: Support alignments larger than PAGE_SIZE.
155        if layout.align() > bindings::PAGE_SIZE {
156            pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n");
157            return Err(AllocError);
158        }
159
160        // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
161        // allocated with this `Allocator`.
162        unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) }
163    }
164}
165
166// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
167// - memory remains valid until it is explicitly freed,
168// - passing a pointer to a valid memory allocation is OK,
169// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
170unsafe impl Allocator for KVmalloc {
171    #[inline]
172    unsafe fn realloc(
173        ptr: Option<NonNull<u8>>,
174        layout: Layout,
175        old_layout: Layout,
176        flags: Flags,
177    ) -> Result<NonNull<[u8]>, AllocError> {
178        // TODO: Support alignments larger than PAGE_SIZE.
179        if layout.align() > bindings::PAGE_SIZE {
180            pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n");
181            return Err(AllocError);
182        }
183
184        // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
185        // allocated with this `Allocator`.
186        unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) }
187    }
188}