diff options
61 files changed, 1054 insertions, 315 deletions
diff --git a/Documentation/gpu/nova/core/todo.rst b/Documentation/gpu/nova/core/todo.rst index 894a1e9c3741..8fdb5bced346 100644 --- a/Documentation/gpu/nova/core/todo.rst +++ b/Documentation/gpu/nova/core/todo.rst @@ -147,7 +147,6 @@ Numerical operations [NUMM] Nova uses integer operations that are not part of the standard library (or not implemented in an optimized way for the kernel). These include: -- Aligning up and down to a power of two, - The "Find Last Set Bit" (`fls` function of the C part of the kernel) operation. diff --git a/drivers/block/rnull.rs b/drivers/block/rnull.rs index d07e76ae2c13..6366da12c5a5 100644 --- a/drivers/block/rnull.rs +++ b/drivers/block/rnull.rs @@ -51,7 +51,7 @@ impl kernel::InPlaceModule for NullBlkModule { .logical_block_size(4096)? .physical_block_size(4096)? .rotational(false) - .build(format_args!("rnullb{}", 0), tagset) + .build(fmt!("rnullb{}", 0), tagset) })(); try_pin_init!(Self { diff --git a/drivers/gpu/drm/drm_panic_qr.rs b/drivers/gpu/drm/drm_panic_qr.rs index 50c286c5cee8..ac27e86c601c 100644 --- a/drivers/gpu/drm/drm_panic_qr.rs +++ b/drivers/gpu/drm/drm_panic_qr.rs @@ -968,7 +968,7 @@ pub unsafe extern "C" fn drm_panic_qr_generate( // nul-terminated string. let url_cstr: &CStr = unsafe { CStr::from_char_ptr(url) }; let segments = &[ - &Segment::Binary(url_cstr.as_bytes()), + &Segment::Binary(url_cstr.to_bytes()), &Segment::Numeric(&data_slice[0..data_len]), ]; match EncodedMsg::new(segments, tmp_slice) { diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 4a702525fff4..e4dc74f2f90a 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -3,6 +3,7 @@ use core::ops::Range; use kernel::prelude::*; +use kernel::ptr::{Alignable, Alignment}; use kernel::sizes::*; use kernel::types::ARef; use kernel::{dev_warn, device}; @@ -130,10 +131,9 @@ impl FbLayout { }; let frts = { - const FRTS_DOWN_ALIGN: u64 = SZ_128K as u64; + const FRTS_DOWN_ALIGN: Alignment = Alignment::new::<SZ_128K>(); const FRTS_SIZE: u64 = SZ_1M as u64; - // TODO[NUMM]: replace with `align_down` once it lands. - let frts_base = (vga_workspace.start & !(FRTS_DOWN_ALIGN - 1)) - FRTS_SIZE; + let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - FRTS_SIZE; frts_base..frts_base + FRTS_SIZE }; diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b5c9786619a9..600cc90b5fab 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use kernel::{device, devres::Devres, error::code::*, pci, prelude::*, sync::Arc}; +use kernel::{device, devres::Devres, error::code::*, fmt, pci, prelude::*, sync::Arc}; use crate::driver::Bar0; use crate::falcon::{gsp::Gsp, sec2::Sec2, Falcon}; @@ -12,7 +12,6 @@ use crate::gfw; use crate::regs; use crate::util; use crate::vbios::Vbios; -use core::fmt; macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => diff --git a/drivers/gpu/nova-core/regs/macros.rs b/drivers/gpu/nova-core/regs/macros.rs index a3e6de1779d4..6b9df4205f46 100644 --- a/drivers/gpu/nova-core/regs/macros.rs +++ b/drivers/gpu/nova-core/regs/macros.rs @@ -149,10 +149,10 @@ macro_rules! register { // TODO[REGA]: display the raw hex value, then the value of all the fields. This requires // matching the fields, which will complexify the syntax considerably... - impl ::core::fmt::Debug for $name { - fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + impl ::kernel::fmt::Debug for $name { + fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { f.debug_tuple(stringify!($name)) - .field(&format_args!("0x{0:x}", &self.0)) + .field(&::kernel::prelude::fmt!("0x{0:x}", &self.0)) .finish() } } diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 5b5d9f38cbb3..091642d6a5a1 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -10,6 +10,7 @@ use kernel::device; use kernel::error::Result; use kernel::pci; use kernel::prelude::*; +use kernel::ptr::{Alignable, Alignment}; /// The offset of the VBIOS ROM in the BAR0 space. const ROM_OFFSET: usize = 0x300000; @@ -177,8 +178,7 @@ impl<'a> Iterator for VbiosIterator<'a> { // Advance to next image (aligned to 512 bytes). self.current_offset += image_size; - // TODO[NUMM]: replace with `align_up` once it lands. - self.current_offset = self.current_offset.next_multiple_of(512); + self.current_offset = self.current_offset.align_up(Alignment::new::<512>())?; Some(Ok(full_image)) } diff --git a/rust/Makefile b/rust/Makefile index bfa915b0e588..23c7ae905bd2 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -98,6 +98,12 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ rustdoc-kernel rustdoc-pin_init + $(Q)grep -Ehro '<a href="srctree/([^"]+)"' $(rustdoc_output) | \ + cut -d'"' -f2 | cut -d/ -f2- | while read f; do \ + if [ ! -e "$(srctree)/$$f" ]; then \ + echo "warning: srctree/ link to $$f does not exist"; \ + fi \ + done $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/ $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/ $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \ @@ -193,12 +199,12 @@ rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ $(obj)/bindings.o FORCE +$(call if_changed,rustc_test_library) -rusttestlib-bindings: private rustc_target_flags = --extern ffi -rusttestlib-bindings: $(src)/bindings/lib.rs rusttestlib-ffi FORCE +rusttestlib-bindings: private rustc_target_flags = --extern ffi --extern pin_init +rusttestlib-bindings: $(src)/bindings/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE +$(call if_changed,rustc_test_library) -rusttestlib-uapi: private rustc_target_flags = --extern ffi -rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi FORCE +rusttestlib-uapi: private rustc_target_flags = --extern ffi --extern pin_init +rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE +$(call if_changed,rustc_test_library) quiet_cmd_rustdoc_test = RUSTDOC T $< @@ -248,7 +254,7 @@ quiet_cmd_rustc_test = $(RUSTC_OR_CLIPPY_QUIET) T $< $(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \ $(rustc_test_run_flags) -rusttest: rusttest-macros rusttest-kernel +rusttest: rusttest-macros rusttest-macros: private rustc_target_flags = --extern proc_macro \ --extern macros --extern kernel --extern pin_init @@ -258,13 +264,6 @@ rusttest-macros: $(src)/macros/lib.rs \ +$(call if_changed,rustc_test) +$(call if_changed,rustdoc_test) -rusttest-kernel: private rustc_target_flags = --extern ffi --extern pin_init \ - --extern build_error --extern macros --extern bindings --extern uapi -rusttest-kernel: $(src)/kernel/lib.rs rusttestlib-ffi rusttestlib-kernel \ - rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ - rusttestlib-uapi rusttestlib-pin_init FORCE - +$(call if_changed,rustc_test) - ifdef CONFIG_CC_IS_CLANG bindgen_c_flags = $(c_flags) else @@ -531,17 +530,19 @@ $(obj)/ffi.o: private skip_gendwarfksyms = 1 $(obj)/ffi.o: $(src)/ffi.rs $(obj)/compiler_builtins.o FORCE +$(call if_changed_rule,rustc_library) -$(obj)/bindings.o: private rustc_target_flags = --extern ffi +$(obj)/bindings.o: private rustc_target_flags = --extern ffi --extern pin_init $(obj)/bindings.o: $(src)/bindings/lib.rs \ $(obj)/ffi.o \ + $(obj)/pin_init.o \ $(obj)/bindings/bindings_generated.rs \ $(obj)/bindings/bindings_helpers_generated.rs FORCE +$(call if_changed_rule,rustc_library) -$(obj)/uapi.o: private rustc_target_flags = --extern ffi +$(obj)/uapi.o: private rustc_target_flags = --extern ffi --extern pin_init $(obj)/uapi.o: private skip_gendwarfksyms = 1 $(obj)/uapi.o: $(src)/uapi/lib.rs \ $(obj)/ffi.o \ + $(obj)/pin_init.o \ $(obj)/uapi/uapi_generated.rs FORCE +$(call if_changed_rule,rustc_library) diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index 0f96af8b9a7f..e13c6f9dd17b 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -34,3 +34,8 @@ # We use const helpers to aid bindgen, to avoid conflicts when constants are # recognized, block generation of the non-helper constants. --blocklist-item ARCH_SLAB_MINALIGN +--blocklist-item ARCH_KMALLOC_MINALIGN + +# Structs should implement `Zeroable` when all of their fields do. +--with-derive-custom-struct .*=MaybeZeroable +--with-derive-custom-union .*=MaybeZeroable diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 84d60635e8a9..4ad9add117ea 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -84,6 +84,7 @@ /* `bindgen` gets confused at certain things. */ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; +const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 474cc98c48a3..0c57cf9b4004 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -31,11 +31,19 @@ #[allow(clippy::undocumented_unsafe_blocks)] #[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] mod bindings_raw { + use pin_init::{MaybeZeroable, Zeroable}; + // Manual definition for blocklisted types. type __kernel_size_t = usize; type __kernel_ssize_t = isize; type __kernel_ptrdiff_t = isize; + // `bindgen` doesn't automatically do this, see + // <https://github.com/rust-lang/rust-bindgen/issues/3196> + // + // SAFETY: `__BindgenBitfieldUnit<Storage>` is a newtype around `Storage`. + unsafe impl<Storage> Zeroable for __BindgenBitfieldUnit<Storage> where Storage: Zeroable {} + // Use glob import here to expose all helpers. // Symbols defined within the module will take precedence to the glob import. pub use super::bindings_helper::*; diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs index 7ae317368b00..37e1161c1298 100644 --- a/rust/kernel/acpi.rs +++ b/rust/kernel/acpi.rs @@ -37,11 +37,8 @@ impl DeviceId { /// Create a new device id from an ACPI 'id' string. #[inline(always)] pub const fn new(id: &'static CStr) -> Self { - build_assert!( - id.len_with_nul() <= Self::ACPI_ID_LEN, - "ID exceeds 16 bytes" - ); - let src = id.as_bytes_with_nul(); + let src = id.to_bytes_with_nul(); + build_assert!(src.len() <= Self::ACPI_ID_LEN, "ID exceeds 16 bytes"); // Replace with `bindings::acpi_device_id::default()` once stabilized for `const`. // SAFETY: FFI type is valid to be zero-initialized. let mut acpi: bindings::acpi_device_id = unsafe { core::mem::zeroed() }; diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index a2c49e5494d3..9c154209423c 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -2,18 +2,11 @@ //! Implementation of the kernel's memory allocation infrastructure. -#[cfg(not(any(test, testlib)))] pub mod allocator; pub mod kbox; pub mod kvec; pub mod layout; -#[cfg(any(test, testlib))] -pub mod allocator_test; - -#[cfg(any(test, testlib))] -pub use self::allocator_test as allocator; - pub use self::kbox::Box; pub use self::kbox::KBox; pub use self::kbox::KVBox; @@ -137,6 +130,14 @@ pub mod flags { /// - Implementers must ensure that all trait functions abide by the guarantees documented in the /// `# Guarantees` sections. pub unsafe trait Allocator { + /// The minimum alignment satisfied by all allocations from this allocator. + /// + /// # Guarantees + /// + /// Any pointer allocated by this allocator is guaranteed to be aligned to `MIN_ALIGN` even if + /// the requested layout has a smaller alignment. + const MIN_ALIGN: usize; + /// Allocate memory based on `layout` and `flags`. /// /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 2692cf90c948..869d9fd69527 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -17,6 +17,8 @@ use crate::alloc::{AllocError, Allocator}; use crate::bindings; use crate::pr_warn; +const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; + /// The contiguous kernel allocator. /// /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also @@ -128,6 +130,8 @@ impl Kmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Kmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + #[inline] unsafe fn realloc( ptr: Option<NonNull<u8>>, @@ -147,6 +151,8 @@ unsafe impl Allocator for Kmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Vmalloc { + const MIN_ALIGN: usize = kernel::page::PAGE_SIZE; + #[inline] unsafe fn realloc( ptr: Option<NonNull<u8>>, @@ -171,6 +177,8 @@ unsafe impl Allocator for Vmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for KVmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + #[inline] unsafe fn realloc( ptr: Option<NonNull<u8>>, diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs deleted file mode 100644 index 90dd987d40e4..000000000000 --- a/rust/kernel/alloc/allocator_test.rs +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users -//! of those types (e.g. `CString`) use kernel allocators for instantiation. -//! -//! In order to allow userspace test cases to make use of such types as well, implement the -//! `Cmalloc` allocator within the `allocator_test` module and type alias all kernel allocators to -//! `Cmalloc`. The `Cmalloc` allocator uses libc's `realloc()` function as allocator backend. - -#![allow(missing_docs)] - -use super::{flags::*, AllocError, Allocator, Flags}; -use core::alloc::Layout; -use core::cmp; -use core::ptr; -use core::ptr::NonNull; - -/// The userspace allocator based on libc. -pub struct Cmalloc; - -pub type Kmalloc = Cmalloc; -pub type Vmalloc = Kmalloc; -pub type KVmalloc = Kmalloc; - -impl Cmalloc { - /// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of - /// `layout`. - pub fn aligned_layout(layout: Layout) -> Layout { - // Note that `layout.size()` (after padding) is guaranteed to be a multiple of - // `layout.align()` which together with the slab guarantees means that `Kmalloc` will return - // a properly aligned object (see comments in `kmalloc()` for more information). - layout.pad_to_align() - } -} - -extern "C" { - #[link_name = "aligned_alloc"] - fn libc_aligned_alloc(align: usize, size: usize) -> *mut crate::ffi::c_void; - - #[link_name = "free"] - fn libc_free(ptr: *mut crate::ffi::c_void); -} - -// SAFETY: -// - memory remains valid until it is explicitly freed, -// - passing a pointer to a valid memory allocation created by this `Allocator` is always OK, -// - `realloc` provides the guarantees as provided in the `# Guarantees` section. -unsafe impl Allocator for Cmalloc { - unsafe fn realloc( - ptr: Option<NonNull<u8>>, - layout: Layout, - old_layout: Layout, - flags: Flags, - ) -> Result<NonNull<[u8]>, AllocError> { - let src = match ptr { - Some(src) => { - if old_layout.size() == 0 { - ptr::null_mut() - } else { - src.as_ptr() - } - } - None => ptr::null_mut(), - }; - - if layout.size() == 0 { - // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` - unsafe { libc_free(src.cast()) }; - - return Ok(NonNull::slice_from_raw_parts( - crate::alloc::dangling_from_layout(layout), - 0, - )); - } - - // ISO C (ISO/IEC 9899:2011) defines `aligned_alloc`: - // - // > The value of alignment shall be a valid alignment supported by the implementation - // [...]. - // - // As an example of the "supported by the implementation" requirement, POSIX.1-2001 (IEEE - // 1003.1-2001) defines `posix_memalign`: - // - // > The value of alignment shall be a power of two multiple of sizeof (void *). - // - // and POSIX-based implementations of `aligned_alloc` inherit this requirement. At the time - // of writing, this is known to be the case on macOS (but not in glibc). - // - // Satisfy the stricter requirement to avoid spurious test failures on some platforms. - let min_align = core::mem::size_of::<*const crate::ffi::c_void>(); - let layout = layout.align_to(min_align).map_err(|_| AllocError)?; - let layout = layout.pad_to_align(); - - // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or - // exceeds the given size and alignment requirements. - let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) }.cast::<u8>(); - let dst = NonNull::new(dst).ok_or(AllocError)?; - - if flags.contains(__GFP_ZERO) { - // SAFETY: The preceding calls to `libc_aligned_alloc` and `NonNull::new` - // guarantee that `dst` points to memory of at least `layout.size()` bytes. - unsafe { dst.as_ptr().write_bytes(0, layout.size()) }; - } - - if !src.is_null() { - // SAFETY: - // - `src` has previously been allocated with this `Allocator`; `dst` has just been - // newly allocated, hence the memory regions do not overlap. - // - both` src` and `dst` are properly aligned and valid for reads and writes - unsafe { - ptr::copy_nonoverlapping( - src, - dst.as_ptr(), - cmp::min(layout.size(), old_layout.size()), - ) - }; - } - - // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` - unsafe { libc_free(src.cast()) }; - - Ok(NonNull::slice_from_raw_parts(dst, layout.size())) - } -} diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 856d05aa60f1..27c4b5a9b61d 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -7,7 +7,6 @@ use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; use super::{AllocError, Allocator, Flags}; use core::alloc::Layout; use core::borrow::{Borrow, BorrowMut}; -use core::fmt; use core::marker::PhantomData; use core::mem::ManuallyDrop; use core::mem::MaybeUninit; @@ -17,6 +16,7 @@ use core::ptr::NonNull; use core::result::Result; use crate::ffi::c_void; +use crate::fmt; use crate::init::InPlaceInit; use crate::types::ForeignOwnable; use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; @@ -290,6 +290,83 @@ where Ok(Self::new(x, flags)?.into()) } + /// Construct a pinned slice of elements `Pin<Box<[T], A>>`. + /// + /// This is a convenient means for creation of e.g. slices of structrures containing spinlocks + /// or mutexes. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{new_spinlock, SpinLock}; + /// + /// struct Inner { + /// a: u32, + /// b: u32, + /// } + /// + /// #[pin_data] + /// struct Example { + /// c: u32, + /// #[pin] + /// |