path: root/drivers/android/binder/page_range.rs

// SPDX-License-Identifier: GPL-2.0

// Copyright (C) 2025 Google LLC.

//! This module has utilities for managing a page range where unused pages may be reclaimed by a
//! vma shrinker.

// To avoid deadlocks, locks are taken in the order:
//
//  1. mmap lock
//  2. spinlock
//  3. lru spinlock
//
// The shrinker will use trylock methods because it locks them in a different order.

use core::{
    marker::PhantomPinned,
    mem::{size_of, size_of_val, MaybeUninit},
    ptr,
};

use kernel::{
    bindings,
    error::Result,
    ffi::{c_ulong, c_void},
    mm::{virt, Mm, MmWithUser},
    new_mutex, new_spinlock,
    page::{Page, PAGE_SHIFT, PAGE_SIZE},
    prelude::*,
    str::CStr,
    sync::{aref::ARef, Mutex, SpinLock},
    task::Pid,
    transmute::FromBytes,
    types::Opaque,
    uaccess::UserSliceReader,
};

/// Represents a shrinker that can be registered with the kernel.
///
/// Each shrinker can be used by many `ShrinkablePageRange` objects.
#[repr(C)]
pub(crate) struct Shrinker {
    inner: Opaque<*mut bindings::shrinker>,
    list_lru: Opaque<bindings::list_lru>,
}

// SAFETY: The shrinker and list_lru are thread safe.
unsafe impl Send for Shrinker {}
// SAFETY: The shrinker and list_lru are thread safe.
unsafe impl Sync for Shrinker {}

impl Shrinker {
    /// Create a new shrinker.
    ///
    /// # Safety
    ///
    /// Before using this shrinker with a `ShrinkablePageRange`, the `register` method must have
    /// been called exactly once, and it must not have returned an error.
    pub(crate) const unsafe fn new() -> Self {
        Self {
            inner: Opaque::uninit(),
            list_lru: Opaque::uninit(),
        }
    }

    /// Register this shrinker with the kernel.
    pub(crate) fn register(&'static self, name: &CStr) -> Result<()> {
        // SAFETY: These fields are not yet used, so it's okay to zero them.
        unsafe {
            self.inner.get().write(ptr::null_mut());
            self.list_lru.get().write_bytes(0, 1);
        }

        // SAFETY: The field is not yet used, so we can initialize it.
        let ret = unsafe { bindings::__list_lru_init(self.list_lru.get(), false, ptr::null_mut()) };
        if ret != 0 {
            return Err(Error::from_errno(ret));
        }

        // SAFETY: The `name` points at a valid c string.
        let shrinker = unsafe { bindings::shrinker_alloc(0, name.as_char_ptr()) };
        if shrinker.is_null() {
            // SAFETY: We initialized it, so its okay to destroy it.
            unsafe { bindings::list_lru_destroy(self.list_lru.get()) };
            return Err(Error::from_errno(ret));
        }

        // SAFETY: We're about to register the shrinker, and these are the fields we need to
        // initialize. (All other fields are already zeroed.)
        unsafe {
            (&raw mut (*shrinker).count_objects).write(Some(rust_shrink_count));
            (&raw mut (*shrinker).scan_objects).write(Some(rust_shrink_scan));
            (&raw mut (*shrinker).private_data).write(self.list_lru.get().cast());
        }

        // SAFETY: The new shrinker has been fully initialized, so we can register it.
        unsafe { bindings::shrinker_register(shrinker) };

        // SAFETY: This initializes the pointer to the shrinker so that we can use it.
        unsafe { self.inner.get().write(shrinker) };

        Ok(())
    }
}

/// A container that manages a page range in a vma.
///
/// The pages can be thought of as an array of booleans of whether the pages are usable. The
/// methods `use_range` and `stop_using_range` set all booleans in a range to true or false
/// respectively. Initially, no pages are allocated. When a page is not used, it is not freed
/// immediately. Instead, it is made available to the memory shrinker to free it if the device is
/// under memory pressure.
///
/// It's okay for `use_range` and `stop_using_range` to race with each other, although there's no
/// way to know whether an index ends up with true or false if a call to `use_range` races with
/// another call to `stop_using_range` on a given index.
///
/// It's also okay for the two methods to race with themselves, e.g. if two threads call
/// `use_range` on the same index, then that's fine and neither call will return until the page is
/// allocated and mapped.
///
/// The methods that read or write to a range require that the page is marked as in use. So it is
/// _not_ okay to call `stop_using_range` on a page that is in use by the methods that read or
/// write to the page.
#[pin_data(PinnedDrop)]
pub(crate) struct ShrinkablePageRange {
    /// Shrinker object registered with the kernel.
    shrinker: &'static Shrinker,
    /// Pid using this page range. Only used as debugging information.
    pid: Pid,
    /// The mm for the relevant process.
    mm: ARef<Mm>,
    /// Used to synchronize calls to `vm_insert_page` and `zap_page_range_single`.
    #[pin]
    mm_lock: Mutex<()>,
    /// Spinlock protecting changes to pages.
    #[pin]
    lock: SpinLock<Inner>,

    /// Must not move, since page info has pointers back.
    #[pin]
    _pin: PhantomPinned,
}

struct Inner {
    /// Array of pages.
    ///
    /// Since this is also accessed by the shrinker, we can't use a `Box`, which asserts exclusive
    /// ownership. To deal with that, we manage it using raw pointers.
    pages: *mut PageInfo,
    /// Length of the `pages` array.
    size: usize,
    /// The address of the vma to insert the pages into.
    vma_addr: usize,
}

// SAFETY: proper locking is in place for `Inner`
unsafe impl Send for Inner {}

type StableMmGuard =
    kernel::sync::lock::Guard<'static, (), kernel::sync::lock::mutex::MutexBackend>;

/// An array element that describes the current state of a page.
///
/// There are three states:
///
///  * Free. The page is None. The `lru` element is not queued.
///  * Available. The page is Some. The `lru` element is queued to the shrinker's lru.
///  * Used. The page is Some. The `lru` element is not queued.
///
/// When an element is available, the shrinker is able to free the page.
#[repr(C)]
struct PageInfo {
    lru: bindings::list_head,
    page: Option<Page>,
    range: *const ShrinkablePageRange,
}

impl PageInfo {
    /// # Safety
    ///
    /// The caller ensures that writing to `me.page` is ok, and that the page is not currently set.
    unsafe fn set_page(me: *mut PageInfo, page: Page) {
        // SAFETY: This pointer offset is in bounds.
        let ptr = unsafe { &raw mut (*me).page };

        // SAFETY: The pointer is valid for writing, so also valid for reading.
        if unsafe { (*ptr).is_some() } {
            pr_err!("set_page called when there is already a page");
            // SAFETY: We will initialize the page again below.
            unsafe { ptr::drop_in_place(ptr) };
        }

        // SAFETY: The pointer is valid for writing.
        unsafe { ptr::write(ptr, Some(page)) };
    }

    /// # Safety
    ///
    /// The caller ensures that reading from `me.page` is ok for the duration of 'a.
    unsafe fn get_page<'a>(me: *const PageInfo) -> Option<&'a Page> {
        // SAFETY: This pointer offset is in bounds.
        let ptr = unsafe { &raw const (*me).page };

        // SAFETY: The pointer is valid for reading.
        unsafe { (*ptr).as_ref() }
    }

    /// # Safety
    ///
    /// The caller ensures that writing to `me.page` is ok for the duration of 'a.
    unsafe fn take_page(me: *mut PageInfo