path: root/drivers/gpu/nova-core/gsp/cmdq.rs

// SPDX-License-Identifier: GPL-2.0

use core::{
    cmp,
    mem,
    sync::atomic::{
        fence,
        Ordering, //
    }, //
};

use kernel::{
    device,
    dma::{
        CoherentAllocation,
        DmaAddress, //
    },
    dma_write,
    io::poll::read_poll_timeout,
    prelude::*,
    sync::aref::ARef,
    time::Delta,
    transmute::{
        AsBytes,
        FromBytes, //
    },
};

use crate::{
    driver::Bar0,
    gsp::{
        fw::{
            GspMsgElement,
            MsgFunction,
            MsgqRxHeader,
            MsgqTxHeader, //
        },
        PteArray,
        GSP_PAGE_SHIFT,
        GSP_PAGE_SIZE, //
    },
    num,
    regs,
    sbuffer::SBufferIter, //
};

/// Trait implemented by types representing a command to send to the GSP.
///
/// The main purpose of this trait is to provide [`Cmdq::send_command`] with the information it
/// needs to send a given command.
///
/// [`CommandToGsp::init`] in particular is responsible for initializing the command directly
/// into the space reserved for it in the command queue buffer.
///
/// Some commands may be followed by a variable-length payload. For these, the
/// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be
/// defined as well.
pub(crate) trait CommandToGsp {
    /// Function identifying this command to the GSP.
    const FUNCTION: MsgFunction;

    /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer.
    type Command: FromBytes + AsBytes;

    /// Error type returned by [`CommandToGsp::init`].
    type InitError;

    /// In-place command initializer responsible for filling the command in the command queue
    /// buffer.
    fn init(&self) -> impl Init<Self::Command, Self::InitError>;

    /// Size of the variable-length payload following the command structure generated by
    /// [`CommandToGsp::init`].
    ///
    /// Most commands don't have a variable-length payload, so this is zero by default.
    fn variable_payload_len(&self) -> usize {
        0
    }

    /// Method initializing the variable-length payload.
    ///
    /// The command buffer is circular, which means that we may need to jump back to its beginning
    /// while in the middle of a command. For this reason, the variable-length payload is
    /// initialized using a [`SBufferIter`].
    ///
    /// This method will receive a buffer of the length returned by
    /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving
    /// unwritten space will lead to an error.
    ///
    /// Most commands don't have a variable-length payload, so this does nothing by default.
    fn init_variable_payload(
        &self,
        _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
    ) -> Result {
        Ok(())
    }
}

/// Trait representing messages received from the GSP.
///
/// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message.
pub(crate) trait MessageFromGsp: Sized {
    /// Function identifying this message from the GSP.
    const FUNCTION: MsgFunction;

    /// Error type returned by [`MessageFromGsp::read`].
    type InitError;

    /// Type containing the raw message to be read from the message queue.
    type Message: FromBytes;

    /// Method reading the message from the message queue and returning it.
    ///
    /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns
    /// it.
    fn read(
        msg: &Self::Message,
        sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>,
    ) -> Result<Self, Self::InitError>;
}

/// Number of GSP pages making the [`Msgq`].
pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f;

/// Circular buffer of a [`Msgq`].
///
/// This area of memory is to be shared between the driver and the GSP to exchange commands or
/// messages.
#[repr(C, align(0x1000))]
#[derive(Debug)]
struct MsgqData {
    data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)],
}

// Annoyingly we are forced to use a literal to specify the alignment of
// `MsgqData`, so check that it corresponds to the actual GSP page size here.
static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE);

/// Unidirectional message queue.
///
/// Contains the data for a message queue, that either the driver or GSP writes to.
///
/// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the
/// read pointer of `rx` actually refers to the `Msgq` owned by the other side.
/// This design ensures that only the driver or GSP ever writes to a given instance of this struct.
#[repr(C)]
// There is no struct defined for this in the open-gpu-kernel-source headers.
// Instead it is defined by code in `GspMsgQueuesInit()`.
struct Msgq {
    /// Header for sending messages, including the write pointer.
    tx: MsgqTxHeader,
    /// Header for receiving messages, including the read pointer.
    rx: MsgqRxHeader,
    /// The message queue proper.
    msgq: MsgqData,
}

/// Structure shared between the driver and the GSP and containing the command and message queues.
#[repr(C)]
struct GspMem {
    /// Self-mapping page table entries.
    ptes: PteArray<{ GSP_PAGE_SIZE / size_of::<u64>() }>,
    /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
    /// write and read pointers that the CPU updates.
    ///
    /// This member is read-only for the GSP.
    cpuq: Msgq,
    /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
    /// write and read pointers that the GSP updates.
    ///
    /// This member is read-only for the driver.
    gspq: Msgq,
}

// SAFETY: These structs don't meet the no-padding requirements of AsBytes but
// that is not a problem because they are not used outside the kernel.
unsafe impl AsBytes for GspMem {}

// SAFETY: These structs don't meet the no-padding requirements of FromBytes but
// that is not a problem because they are not used outside the kernel.
unsafe impl FromBytes for GspMem {}

/// Wrapper around [`GspMem`] to share it with the GPU using a [`CoherentAllocation`].
///
/// This provides the low-level functionality to communicate with the GSP, including allocation of
/// queue space to write messages to and management of read/write pointers.
///
/// This is shared with the GSP, with clear ownership rules regarding the command queues:
///
/// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write
///   pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`].
/// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read
///   pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`].
struct DmaGspMem(CoherentAllocation<GspMem>);

impl DmaGspMem {
    /// Allocate a new instance and map it for `dev`.
    fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
        const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>();
        const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>();

        let gsp_mem =
            CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?;
        dma_write!(gsp_mem[0].ptes = PteArray::new(gsp_mem.dma_handle())?)?;
        dma_write!(gsp_mem[0].cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES))?;
        dma_write!(gsp_mem[0].cpuq.rx = MsgqRxHeader::new())?;

        Ok(Self(gsp_mem))
    }

    /// Returns the region of the CPU message queue that the driver is currently allowed to write
    /// to.
    ///
    /// As the message queue is a circular buffer, the region may be discontiguous in memory. In
    /// that case the second slice will have a non-zero length.
    fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) {
        let tx = self.cpu_write_ptr() as usize;
        let rx = self.gsp_read_ptr() as usize;

        // SAFETY:
        // - The `CoherentAllocation` contains exactly one object.
        // - We will only access the driver-owned part of the shared memory.
        // - Per the safety statement of the function, no concurrent access will be performed.
        let gsp_mem = &mut unsafe { self.0.as_slice_mut(0, 1) }.unwrap()[0];
        // PANIC: per the invariant of `cpu_write_ptr`, `tx` is `<= MSGQ_NUM_PAGES`.
        let (before_tx, after_tx) = gsp_mem.cpuq.msgq.data.split_at_mut(tx);

        if rx <= tx {
            // The area from `tx` up to the end of the ring, and from the beginning of the ring up
            // to `rx`, minus one unit, belongs to the driver.
            if rx == 0 {
                let last = after_tx.len() - 1;
                (&mut after_tx[..last], &mut before_tx[0..