aboutsummaryrefslogtreecommitdiff
path: root/rust
diff options
context:
space:
mode:
authorDanilo Krummrich <dakr@kernel.org>2026-07-13 23:50:39 +0200
committerDanilo Krummrich <dakr@kernel.org>2026-07-13 23:50:39 +0200
commitb07fc8d60bd30caaba4d293929459780166da194 (patch)
tree4f47c377f5b9c8078f02ef883322719f4109b167 /rust
parenta56a92be1fa9c6e747fc754c90734ed90cd36670 (diff)
parent11a4784f902ebf3e674dbaf07dbec9a37aabb5e4 (diff)
Merge patch series "rust: I/O type generalization and projection"
Gary Guo <gary@garyguo.net> says: This series presents a major rework of I/O types, as a summary: - Make I/O regions typed. The existing untyped region still exists with a dynamically sized `Region` type. - Create I/O view types to represent subregion of a full I/O region mapped. A projection macro is added to allow safely create such subviews. - Split I/O traits, make I/O views play a central role, avoid duplicate monomorphization and less `unsafe` code. - Add a `SysMem` backend, and make `Coherent` implement `Io`. - Add copying methods (memcpy_{from,to}io and friends). This series generalize `Mmio` type from just an untyped region to typed representations (so `MmioRaw<T>` is `__iomem *T`). This allows us to remove the `IoKnownSize` trait; the information is sourced from just the pointer from the `KnownSize` trait instead. Building on top of that, `Mmio` and `ConfigSpace` have been converted to typed views of I/O regions rather than just a big chunk of untyped I/O memory. These changes made it possible to implement `Io` trait for `Coherent<T>`. Shared system memory, `SysMem` is also added to the series, given it similarity in implementation compared to `Coherent`. In fact, the series use `SysMem` to implement `Coherent`'s I/O methods. Built on these generalization, this series add `io_project!()`. `io_project!()` performs a safe way to project a bigger view to a small subviews, and some Nova code has been converted in this series to demonstrate cleanups possible with this addition. New `io_read!()`, `io_write!()` has been added that supersedes `dma_read!()`, `dma_write!()` macro. Although, they work for primitives only (to be exact, types that the backend is `IoCapable` of). One feature that was lost from the old `dma_read!()` and `dma_write!()` series was the ability to read/write a large structs. However, the semantics was unclear to begin with, as there was no guarantee about their atomicity even for structs that were small enough to fit in u32. Suggested-by: Danilo Krummrich <dakr@kernel.org> Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Generic.20I.2FO.20backends/near/571198078 Link: https://patch.msgid.link/20260706-io_projection-v6-0-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich <dakr@kernel.org>
Diffstat (limited to 'rust')
-rw-r--r--rust/helpers/io.c13
-rw-r--r--rust/kernel/devres.rs24
-rw-r--r--rust/kernel/dma.rs281
-rw-r--r--rust/kernel/io.rs1502
-rw-r--r--rust/kernel/io/mem.rs29
-rw-r--r--rust/kernel/io/poll.rs6
-rw-r--r--rust/kernel/io/register.rs44
-rw-r--r--rust/kernel/lib.rs3
-rw-r--r--rust/kernel/pci.rs1
-rw-r--r--rust/kernel/pci/io.rs166
-rw-r--r--rust/kernel/ptr.rs12
11 files changed, 1525 insertions, 556 deletions
diff --git a/rust/helpers/io.c b/rust/helpers/io.c
index 397810864a24..7ed9a4f77f1b 100644
--- a/rust/helpers/io.c
+++ b/rust/helpers/io.c
@@ -19,6 +19,19 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr)
iounmap(addr);
}
+__rust_helper void rust_helper_memcpy_fromio(void *dst,
+ const volatile void __iomem *src,
+ size_t count)
+{
+ memcpy_fromio(dst, src, count);
+}
+
+__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst,
+ const void *src, size_t count)
+{
+ memcpy_toio(dst, src, count);
+}
+
__rust_helper u8 rust_helper_readb(const void __iomem *addr)
{
return readb(addr);
diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs
index b7c075a39ba4..20f94030f977 100644
--- a/rust/kernel/devres.rs
+++ b/rust/kernel/devres.rs
@@ -70,17 +70,19 @@ struct Inner<T> {
/// devres::Devres,
/// io::{
/// Io,
-/// IoKnownSize,
+/// IoBase,
/// Mmio,
/// MmioRaw,
-/// PhysAddr, //
+/// MmioBackend,
+/// PhysAddr,
+/// Region, //
/// },
/// prelude::*,
/// };
/// use core::ops::Deref;
///
/// // See also [`pci::Bar`] for a real example.
-/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>);
+/// struct IoMem<const SIZE: usize>(MmioRaw<Region<SIZE>>);
///
/// impl<const SIZE: usize> IoMem<SIZE> {
/// /// # Safety
@@ -95,7 +97,7 @@ struct Inner<T> {
/// return Err(ENOMEM);
/// }
///
-/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?))
+/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?))
/// }
/// }
///
@@ -106,12 +108,13 @@ struct Inner<T> {
/// }
/// }
///
-/// impl<const SIZE: usize> Deref for IoMem<SIZE> {
-/// type Target = Mmio<SIZE>;
+/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<SIZE> {
+/// type Backend = MmioBackend;
+/// type Target = Region<SIZE>;
///
-/// fn deref(&self) -> &Self::Target {
+/// fn as_view(self) -> Mmio<'a, Region<SIZE>> {
/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`.
-/// unsafe { Mmio::from_raw(&self.0) }
+/// unsafe { Mmio::from_raw(self.0) }
/// }
/// }
/// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> {
@@ -299,10 +302,7 @@ impl<T: Send + 'static> Devres<T> {
/// use kernel::{
/// device::Core,
/// devres::Devres,
- /// io::{
- /// Io,
- /// IoKnownSize, //
- /// },
+ /// io::Io,
/// pci, //
/// };
///
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 200def84fb69..e275f2562a5b 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -14,14 +14,22 @@ use crate::{
},
error::to_result,
fs::file,
+ io::{
+ IoBackend,
+ IoBase,
+ IoCapable,
+ IoCopyable,
+ SysMem,
+ SysMemBackend, //
+ },
prelude::*,
ptr::KnownSize,
sync::aref::ARef,
transmute::{
AsBytes,
FromBytes, //
- }, //
- uaccess::UserSliceWriter,
+ },
+ uaccess::UserSliceWriter, //
};
use core::{
ops::{
@@ -654,52 +662,6 @@ impl<T: KnownSize + ?Sized> Coherent<T> {
// SAFETY: per safety requirement.
unsafe { &mut *self.as_mut_ptr() }
}
-
- /// Reads the value of `field` and ensures that its type is [`FromBytes`].
- ///
- /// # Safety
- ///
- /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is
- /// validated beforehand.
- ///
- /// Public but hidden since it should only be used from [`dma_read`] macro.
- #[doc(hidden)]
- pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {
- // SAFETY:
- // - By the safety requirements field is valid.
- // - Using read_volatile() here is not sound as per the usual rules, the usage here is
- // a special exception with the following notes in place. When dealing with a potential
- // race from a hardware or code outside kernel (e.g. user-space program), we need that
- // read on a valid memory is not UB. Currently read_volatile() is used for this, and the
- // rationale behind is that it should generate the same code as READ_ONCE() which the
- // kernel already relies on to avoid UB on data races. Note that the usage of
- // read_volatile() is limited to this particular case, it cannot be used to prevent
- // the UB caused by racing between two kernel functions nor do they provide atomicity.
- unsafe { field.read_volatile() }
- }
-
- /// Writes a value to `field` and ensures that its type is [`AsBytes`].
- ///
- /// # Safety
- ///
- /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is
- /// validated beforehand.
- ///
- /// Public but hidden since it should only be used from [`dma_write`] macro.
- #[doc(hidden)]
- pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) {
- // SAFETY:
- // - By the safety requirements field is valid.
- // - Using write_volatile() here is not sound as per the usual rules, the usage here is
- // a special exception with the following notes in place. When dealing with a potential
- // race from a hardware or code outside kernel (e.g. user-space program), we need that
- // write on a valid memory is not UB. Currently write_volatile() is used for this, and the
- // rationale behind is that it should generate the same code as WRITE_ONCE() which the
- // kernel already relies on to avoid UB on data races. Note that the usage of
- // write_volatile() is limited to this particular case, it cannot be used to prevent
- // the UB caused by racing between two kernel functions nor do they provide atomicity.
- unsafe { field.write_volatile(val) }
- }
}
impl<T: AsBytes + FromBytes> Coherent<T> {
@@ -1133,84 +1095,153 @@ unsafe impl Send for CoherentHandle {}
// plain `Copy` values.
unsafe impl Sync for CoherentHandle {}
-/// Reads a field of an item from an allocated region of structs.
+/// View type for `Coherent`.
///
-/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating
-/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!).
-///
-/// # Examples
-///
-/// ```
-/// use kernel::device::Device;
-/// use kernel::dma::{attrs::*, Coherent};
-///
-/// struct MyStruct { field: u32, }
-///
-/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
-/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
-/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
-/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
-///
-/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
-/// let whole = kernel::dma_read!(alloc, [try: 2]);
-/// let field = kernel::dma_read!(alloc, [panic: 1].field);
-/// # Ok::<(), Error>(()) }
-/// ```
-#[macro_export]
-macro_rules! dma_read {
- ($dma:expr, $($proj:tt)*) => {{
- let dma = &$dma;
- let ptr = $crate::ptr::project!(
- $crate::dma::Coherent::as_ptr(dma), $($proj)*
- );
- // SAFETY: The pointer created by the projection is within the DMA region.
- unsafe { $crate::dma::Coherent::field_read(dma, ptr) }
- }};
+/// This is same as [`SysMem`] but with additional information that allows handing out a DMA handle.
+pub struct CoherentView<'a, T: ?Sized> {
+ cpu_addr: SysMem<'a, T>,
+ dma_handle: DmaAddress,
}
-/// Writes to a field of an item from an allocated region of structs.
-///
-/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression
-/// evaluating to a [`Coherent`], `proj` is a
-/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the
-/// projected location.
-///
-/// # Examples
-///
-/// ```
-/// use kernel::device::Device;
-/// use kernel::dma::{attrs::*, Coherent};
-///
-/// struct MyStruct { member: u32, }
-///
-/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
-/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
-/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
-/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
-///
-/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
-/// kernel::dma_write!(alloc, [try: 2].member, 0xf);
-/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf });
-/// # Ok::<(), Error>(()) }
-/// ```
-#[macro_export]
-macro_rules! dma_write {
- (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{
- let dma = &$dma;
- let ptr = $crate::ptr::project!(
- mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)*
- );
- let val = $val;
- // SAFETY: The pointer created by the projection is within the DMA region.
- unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) }
- }};
- (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
- $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*])
- };
- (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
- $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*])
- };
- ($dma:expr, $($rest:tt)*) => {
- $crate::dma_write!(@parse [$dma] [] [$($rest)*])
- };
+impl<T: ?Sized> Copy for CoherentView<'_, T> {}
+impl<T: ?Sized> Clone for CoherentView<'_, T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+impl<'a, T: ?Sized> CoherentView<'a, T> {
+ /// Erase the DMA handle information and obtain a [`SysMem`] view of the same memory region.
+ #[inline]
+ pub fn as_sys_mem(self) -> SysMem<'a, T> {
+ self.cpu_addr
+ }
+
+ /// Returns a DMA handle which may be given to the device as the DMA address base of the region.
+ #[inline]
+ pub fn dma_handle(self) -> DmaAddress {
+ self.dma_handle
+ }
+
+ /// Returns a reference to the data in the region.
+ ///
+ /// # Safety
+ ///
+ /// * Callers must ensure that the device does not read/write to/from memory while the returned
+ /// reference is live.
+ /// * Callers must ensure that this call does not race with a write (including call to `as_mut`)
+ /// to the same region while the returned reference is live.
+ #[inline]
+ pub unsafe fn as_ref(self) -> &'a T {
+ // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
+ // safety requirement.
+ unsafe { &*self.cpu_addr.as_ptr() }
+ }
+
+ /// Returns a mutable reference to the data in the region.
+ ///
+ /// # Safety
+ ///
+ /// * Callers must ensure that the device does not read/write to/from memory while the returned
+ /// reference is live.
+ /// * Callers must ensure that this call does not race with a read (including call to `as_ref`)
+ /// or write (including call to `as_mut`) to the same region while the returned reference is
+ /// live.
+ #[inline]
+ pub unsafe fn as_mut(self) -> &'a mut T {
+ // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
+ // safety requirement.
+ unsafe { &mut *self.cpu_addr.as_ptr() }
+ }
+}
+
+/// `IoBackend` implementation for `Coherent`.
+pub struct CoherentIoBackend;
+
+impl IoBackend for CoherentIoBackend {
+ type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>;
+
+ #[inline]
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+ SysMemBackend::as_ptr(view.cpu_addr)
+ }
+
+ #[inline]
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U> {
+ let offset = ptr.addr() - view.cpu_addr.as_ptr().addr();
+ // CAST: The offset DMA address can never overflow.
+ let dma_handle = view.dma_handle + offset as DmaAddress;
+ CoherentView {
+ dma_handle,
+ // SAFETY: Per safety requirement.
+ cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) },
+ }
+ }
+}
+
+impl<T> IoCapable<T> for CoherentIoBackend
+where
+ SysMemBackend: IoCapable<T>,
+{
+ #[inline]
+ fn io_read<'a>(view: Self::View<'a, T>) -> T {
+ SysMemBackend::io_read(view.cpu_addr)
+ }
+
+ #[inline]
+ fn io_write<'a>(view: Self::View<'a, T>, value: T) {
+ SysMemBackend::io_write(view.cpu_addr, value)
+ }
+}
+
+impl IoCopyable for CoherentIoBackend {
+ #[inline]
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+ // SAFETY: Per safety requirement.
+ unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) }
+ }
+
+ #[inline]
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+ // SAFETY: Per safety requirement.
+ unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) }
+ }
+
+ #[inline]
+ fn copy_read<T: zerocopy::FromBytes>(view: Self::View<'_, T>) -> T {
+ SysMemBackend::copy_read(view.cpu_addr)
+ }
+
+ #[inline]
+ fn copy_write<T: zerocopy::IntoBytes>(view: Self::View<'_, T>, value: T) {
+ SysMemBackend::copy_write(view.cpu_addr, value)
+ }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> {
+ type Backend = CoherentIoBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> CoherentView<'a, Self::Target> {
+ self
+ }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> {
+ type Backend = CoherentIoBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> CoherentView<'a, Self::Target> {
+ CoherentView {
+ // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory.
+ cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) },
+ dma_handle: self.dma_handle,
+ }
+ }
}
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index fcc7678fd9e3..95f46bb75f9e 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -4,9 +4,18 @@
//!
//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h)
+use core::{
+ marker::PhantomData,
+ mem::MaybeUninit, //
+};
+
use crate::{
bindings,
- prelude::*, //
+ prelude::*,
+ ptr::{
+ Alignment,
+ KnownSize, //
+ }, //
};
pub mod mem;
@@ -31,128 +40,226 @@ pub type PhysAddr = bindings::phys_addr_t;
/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
pub type ResourceSize = bindings::resource_size_t;
+/// Untyped I/O region.
+///
+/// This type can be used when an I/O region without known type information has a compile-time known
+/// minimum size (and a runtime known actual size).
+///
+/// # Invariants
+///
+/// - Size of the region is at least as large as the `SIZE` generic parameter.
+/// - Size of the region is multiple of 4.
+#[repr(C, align(4))]
+#[derive(FromBytes)]
+pub struct Region<const SIZE: usize = 0> {
+ inner: [u8],
+}
+
+impl<const SIZE: usize> Region<SIZE> {
+ /// Create a raw mutable pointer from given base address and size.
+ ///
+ /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should
+ /// be 4-byte aligned to uphold the type invariant.
+ ///
+ /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer
+ /// that does not uphold the type invariants. However such pointers are not valid.
+ #[inline]
+ pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self {
+ core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE>
+ }
+
+ /// Create a raw mutable pointer from given base address and size.
+ ///
+ /// The alignment of `base` is checked, and `size` is checked against the minimum size specified
+ /// via const generics.
+ #[inline]
+ pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> {
+ if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) {
+ return Err(EINVAL);
+ }
+
+ Ok(Self::ptr_from_raw_parts_mut(base, size))
+ }
+}
+
+impl<const SIZE: usize> KnownSize for Region<SIZE> {
+ const MIN_SIZE: usize = SIZE;
+ // Alignment of 4 is the most common; different base types can be added once required.
+ const MIN_ALIGN: Alignment = Alignment::new::<4>();
+
+ #[inline(always)]
+ fn size(p: *const Self) -> usize {
+ (p as *const [u8]).len()
+ }
+}
+
+// SAFETY:
+// - Values read from I/O are always treated as initialized.
+// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding
+// free.
+//
+// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type
+// invariant which the macro does not know.
+unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> {
+ #[inline]
+ #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings.
+ fn only_derive_is_allowed_to_implement_this_trait() {}
+}
+
/// Raw representation of an MMIO region.
///
+/// `MmioRaw<T>` is equivalent to `T __iomem *` in C.
+///
/// By itself, the existence of an instance of this structure does not provide any guarantees that
/// the represented MMIO region does exist or is properly mapped.
///
/// Instead, the bus specific MMIO implementation must convert this raw representation into an
/// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio`
/// structure any guarantees are given.
-pub struct MmioRaw<const SIZE: usize = 0> {
- addr: usize,
- maxsize: usize,
+pub struct MmioRaw<T: ?Sized> {
+ /// Pointer is in I/O address space.
+ ///
+ /// The provenance does not matter, only the address and metadata do.
+ ptr: *mut T,
}
-impl<const SIZE: usize> MmioRaw<SIZE> {
- /// Returns a new `MmioRaw` instance on success, an error otherwise.
- pub fn new(addr: usize, maxsize: usize) -> Result<Self> {
- if maxsize < SIZE {
- return Err(EINVAL);
+impl<T: ?Sized> Copy for MmioRaw<T> {}
+impl<T: ?Sized> Clone for MmioRaw<T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+// SAFETY: `MmioRaw` is just an address, so is thread-safe.
+unsafe impl<T: ?Sized> Send for MmioRaw<T> {}
+// SAFETY: `MmioRaw` is just an address, so is thread-safe.
+unsafe impl<T: ?Sized> Sync for MmioRaw<T> {}
+
+impl<T> MmioRaw<T> {
+ /// Create a `MmioRaw` from address.
+ #[inline]
+ pub fn new(addr: usize) -> Self {
+ Self {
+ ptr: core::ptr::without_provenance_mut(addr),
}
+ }
+}
- Ok(Self { addr, maxsize })
+impl<const SIZE: usize> MmioRaw<Region<SIZE>> {
+ /// Create a `MmioRaw` representing a I/O region with given size.
+ ///
+ /// The size is checked against the minimum size specified via const generics.
+ #[inline]
+ pub fn new_region(addr: usize, size: usize) -> Result<Self> {
+ Ok(Self {
+ ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?,
+ })
}
+}
+impl<T: ?Sized + KnownSize> MmioRaw<T> {
/// Returns the base address of the MMIO region.
#[inline]
pub fn addr(&self) -> usize {
- self.addr
+ self.ptr.addr()
}
- /// Returns the maximum size of the MMIO region.
+ /// Returns the size of the MMIO region.
#[inline]
- pub fn maxsize(&self) -> usize {
- self.maxsize
+ pub fn size(&self) -> usize {
+ KnownSize::size(self.ptr)
}
}
-/// IO-mapped memory region.
-///
-/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the
-/// mapping, performing an additional region request etc.
-///
-/// # Invariant
-///
-/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size
-/// `maxsize`.
-///
-/// # Examples
-///
-/// ```no_run
-/// use kernel::{
-/// bindings,
-/// ffi::c_void,
-/// io::{
-/// Io,
-/// IoKnownSize,
-/// Mmio,
-/// MmioRaw,
-/// PhysAddr,
-/// },
-/// };
-/// use core::ops::Deref;
-///
-/// // See also `pci::Bar` for a real example.
-/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>);
-///
-/// impl<const SIZE: usize> IoMem<SIZE> {
-/// /// # Safety
-/// ///
-/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs
-/// /// virtual address space.
-/// unsafe fn new(paddr: usize) -> Result<Self>{
-/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is
-/// // valid for `ioremap`.
-/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) };
-/// if addr.is_null() {
-/// return Err(ENOMEM);
-/// }
-///
-/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?))
-/// }
-/// }
-///
-/// impl<const SIZE: usize> Drop for IoMem<SIZE> {
-/// fn drop(&mut self) {
-/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`.
-/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); };
-/// }
-/// }
-///
-/// impl<const SIZE: usize> Deref for IoMem<SIZE> {
-/// type Target = Mmio<SIZE>;
-///
-/// fn deref(&self) -> &Self::Target {
-/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`.
-/// unsafe { Mmio::from_raw(&self.0) }
-/// }
-/// }
-///
-///# fn no_run() -> Result<(), Error> {
-/// // SAFETY: Invalid usage for example purposes.
-/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? };
-/// iomem.write32(0x42, 0x0);
-/// assert!(iomem.try_write32(0x42, 0x0).is_ok());
-/// assert!(iomem.try_write32(0x42, 0x4).is_err());
-/// # Ok(())
-/// # }
-/// ```
-#[repr(transparent)]
-pub struct Mmio<const SIZE: usize = 0>(MmioRaw<SIZE>);
-
-/// Checks whether an access of type `U` at the given `offset`
+/// Checks whether an access of type `U` at the given `base` and the given `offset`
/// is valid within this region.
+///
+/// The `base` is used for alignment checking only. This can be set to 0 to skip the check.
#[inline]
-const fn offset_valid<U>(offset: usize, size: usize) -> bool {
- let type_size = core::mem::size_of::<U>();
- if let Some(end) = offset.checked_add(type_size) {
- end <= size && offset % type_size == 0
+const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool {
+ if let Some(end) = offset.checked_add(size_of::<U>()) {
+ end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0)
} else {
false
}
}
+/// Returns a view for a given `offset`, performing compile-time bound checks.
+// Always inline to optimize out error path of `build_assert`.
+#[inline(always)]
+fn io_view_assert<'a, IO: Io<'a>, U>(
+ this: IO,
+ offset: usize,
+) -> <IO::Backend as IoBackend>::View<'a, U> {
+ // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and
+ // ensure alignment by checking that the alignment of `U` is smaller or equal to the
+ // alignment of `IO::Target`.
+ const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize());
+ build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE));
+
+ let view = this.as_view();
+ let ptr = IO::Backend::as_ptr(view);
+ let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
+ // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
+ // valid projection.
+ unsafe { IO::Backend::project_view(view, projected_ptr) }
+}
+
+/// Returns a view for a given `offset`, performing runtime bound checks.
+#[inline]
+fn io_view<'a, IO: Io<'a>, U>(
+ this: IO,
+ offset: usize,
+) -> Result<<IO::Backend as IoBackend>::View<'a, U>> {
+ let view = this.as_view();
+ let ptr = IO::Backend::as_ptr(view);
+
+ if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) {
+ return Err(EINVAL);
+ }
+
+ let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
+ // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
+ // valid projection.
+ Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
+}
+
+/// I/O backends.
+///
+/// This is an abstract representation to be implemented by arbitrary I/O
+/// backends (e.g. MMIO, PCI config space, etc.).
+///
+/// The base trait only defines the projection operations; which I/O methods are available depends
+/// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions,
+/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI
+/// configuration space, u8, u16, and u32 are supported but u64 is not.
+///
+/// This trait is separate from the `Io` trait as multiple different I/O types may share the same
+/// operation.
+pub trait IoBackend {
+ /// View type for this I/O backend.
+ type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>;
+
+ /// Convert a `view` to a raw pointer for projection.
+ ///
+ /// The returned pointer is private implementation detail of the backend; it is likely not
+ /// valid. It should not be dereferenced.
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T;
+
+ /// Project `view` to its subregion indicated by `ptr`.
+ ///
+ /// If input `view` is valid, returned view must also be valid.
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a projection of `Self::as_ptr(view)`.
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U>;
+}
+
/// Trait indicating that an I/O backend supports operations of a certain type and providing an
/// implementation for these operations.
///
@@ -161,20 +268,75 @@ const fn offset_valid<U>(offset: usize, size: usize) -> bool {
/// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`,
/// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit
/// system might implement all four.
-pub trait IoCapable<T> {
- /// Performs an I/O read of type `T` at `address` and returns the result.
+pub trait IoCapable<T>: IoBackend {
+ /// Performs an I/O read of type `T` at `view` and returns the result.
+ fn io_read<'a>(view: Self::View<'a, T>) -> T;
+
+ /// Performs an I/O write of `value` at `view`.
+ fn io_write<'a>(view: Self::View<'a, T>, value: T);
+}
+
+/// Trait indicating that an I/O backend supports memory copy operations.
+pub trait IoCopyable: IoBackend {
+ /// Copy contents of `view` to `buffer`.
///
/// # Safety
///
- /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`.
- unsafe fn io_read(&self, address: usize) -> T;
+ /// - `buffer` is valid for volatile write for `view.size()` bytes.
+ /// - `buffer` should not overlap with `view`.
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8);
- /// Performs an I/O write of `value` at `address`.
+ /// Copy contents from `buffer` to `view`.
///
/// # Safety
///
- /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`.
- unsafe fn io_write(&self, value: T, address: usize);
+ /// - `buffer` is valid for volatile read for `view.size()` bytes.
+ /// - `buffer` should not overlap with `view`.
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8);
+
+ /// Copy from `view` and return the value.
+ #[inline]
+ fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
+ // Project `self` to `[u8]`.
+ let ptr = Self::as_ptr(view);
+ // SAFETY: This is a identity projection.
+ let slice_view = unsafe {
+ Self::project_view(
+ view,
+ core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
+ )
+ };
+
+ let mut buf = MaybeUninit::<T>::uninit();
+ // SAFETY:
+ // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes.
+ // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`.
+ unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) };
+ // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid.
+ unsafe { buf.assume_init() }
+ }
+
+ /// Copy `value` to `view`.
+ ///
+ /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`].
+ #[inline]
+ fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
+ // Project `self` to `[u8]`.
+ let ptr = Self::as_ptr(view);
+ // SAFETY: This is a identity projection.
+ let slice_view = unsafe {
+ Self::project_view(
+ view,
+ core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
+ )
+ };
+
+ // SAFETY:
+ // - `&raw const value` is valid for read for `size_of::<T>()` bytes.
+ // - `value` is local so `&raw const value` cannot overlap with `slice_view`.
+ unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) };
+ core::mem::forget(value);
+ }
}
/// Describes a given I/O location: its offset, width, and type to convert the raw value from and
@@ -186,15 +348,16 @@ pub trait IoCapable<T> {
/// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`]
/// macro).
///
-/// An `IoLoc<T>` carries three pieces of information:
+/// An `IoLoc<Base, T>` carries the following pieces of information:
///
+/// - The valid `Base` to operate on. For most registers, this should be [`Region`].
/// - The offset to access (returned by [`IoLoc::offset`]),
/// - The width of the access (determined by [`IoLoc::IoType`]),
/// - The type `T` in which the raw data is returned or provided.
///
/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type
/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`).
-pub trait IoLoc<T> {
+pub trait IoLoc<Base: ?Sized, T> {
/// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset).
type IoType: Into<T> + F